Index: /branches/czw_branch/20170908/Ohana/Makefile.in =================================================================== --- /branches/czw_branch/20170908/Ohana/Makefile.in (revision 40476) +++ /branches/czw_branch/20170908/Ohana/Makefile.in (revision 40477) @@ -51,4 +51,5 @@ relphot \ relastro \ +skycalc \ shell \ tools \ @@ -61,5 +62,4 @@ fixcat \ gophot \ -skycalc \ getusno \ lightcurve \ Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/FilterStars.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/FilterStars.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/FilterStars.c (revision 40477) @@ -73,5 +73,6 @@ outcat->measure[N].airmass = airmass (image[0].secz, outcat->average[N].R, outcat->average[N].D, image[0].sidtime, image[0].latitude); outcat->measure[N].az = azimuth (15.0*image[0].sidtime - outcat->average[N].R, outcat->average[N].D, image[0].latitude); - outcat->measure[N].Mcal = image[0].Mcal; + outcat->measure[N].McalPSF = image[0].McalPSF; + outcat->measure[N].McalAPER= image[0].McalAPER; outcat->measure[N].t = image[0].tzero + 1e-4*outcat->measure[N].Yccd*image[0].trate; /* trate is in 0.1 msec / row */ outcat->measure[N].dt = MTIME; Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/LoadDataPMM.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/LoadDataPMM.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/LoadDataPMM.c (revision 40477) @@ -234,12 +234,14 @@ // altaz (&alt, &az, 15.0*image[0].sidtime - image[0].coords.crval1, image[0].coords.crval2, Latitude); + // secz is in units of airmass image[0].trate = 0.0; image[0].secz = 1.0; image[0].ccdnum = 0; - // secz is in units milli-airmass - image[0].Mcal = 0.0; - image[0].Xm = NAN_S_SHORT; - image[0].flags = 0; + image[0].McalPSF = 0.0; + image[0].McalAPER = 0.0; + image[0].McalChiSq = NAN; + image[0].dMcal = NAN; + image[0].flags = 0; image[0].nstar = 0; Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadImageHeader.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadImageHeader.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadImageHeader.c (revision 40477) @@ -265,9 +265,9 @@ } - /* secz is in units milli-airmass */ - image[0].Mcal = ZERO_POINT_OFFSET; - image[0].dMcal = ZERO_POINT_ERROR; - image[0].Xm = NAN_S_SHORT; - image[0].flags = 0; + image[0].McalPSF = ZERO_POINT_OFFSET; + image[0].McalAPER = ZERO_POINT_OFFSET; + image[0].dMcal = ZERO_POINT_ERROR; + image[0].McalChiSq = NAN; + image[0].flags = 0; /* find expected number of stars */ Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadSDSSHeader.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadSDSSHeader.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadSDSSHeader.c (revision 40477) @@ -47,6 +47,8 @@ // secz is in units milli-airmass - image[0].Mcal = 0.0; - image[0].Xm = NAN_S_SHORT; + image[0].McalPSF = 0.0; + image[0].McalAPER = 0.0; + image[0].McalChiSq = NAN; + image[0].dMcal = NAN; image[0].code = 0; memset (image[0].dummy, 0, sizeof(image[0].dummy)); Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadStarsSDSS.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadStarsSDSS.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadStarsSDSS.c (revision 40477) @@ -258,12 +258,14 @@ altaz (&alt, &az, 15.0*images[N].sidtime - images[N].coords.crval1, images[N].coords.crval2, Latitude); + // secz is in units of airmass images[N].trate = clockRate * 1e-4; images[N].secz = catalog->measure[0].airmass; images[N].ccdnum = camcol; - // secz is in units milli-airmass - images[N].Mcal = 0.0; - images[N].Xm = NAN_S_SHORT; - images[N].flags = 0; + images[N].McalPSF = 0.0; + images[N].McalAPER = 0.0; + images[N].McalChiSq = NAN; + images[N].dMcal = NAN; + images[N].flags = 0; images[N].nstar = Nstars; Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadStarsUKIRT.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadStarsUKIRT.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/ReadStarsUKIRT.c (revision 40477) @@ -196,5 +196,6 @@ catalog->measure[i].FWy = ToShortPixels(seeing); - catalog->measure[i].Mcal = Mcal; + catalog->measure[i].McalPSF = Mcal; + catalog->measure[i].McalAPER = Mcal; catalog->measure[i].detID = i; @@ -251,7 +252,9 @@ // secz is in units milli-airmass - images[N].Mcal = Mcal; - images[N].Xm = NAN_S_SHORT; - images[N].flags = 0; + images[N].McalPSF = Mcal; + images[N].McalAPER = Mcal; + images[N].McalChiSq = NAN; + images[N].dMcal = NAN; + images[N].flags = 0; images[N].nstar = Nstars; Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/SEDfit.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/SEDfit.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/SEDfit.c (revision 40477) @@ -172,5 +172,6 @@ outcat[0].measure[Nmeas].M = table[0].row[minFit.row][0].mags[n] + minFit.Md; outcat[0].measure[Nmeas].dM = 0.0; - outcat[0].measure[Nmeas].Mcal = 0; + outcat[0].measure[Nmeas].McalPSF = 0; + outcat[0].measure[Nmeas].McalAPER = 0; outcat[0].measure[Nmeas].t = TIMEREF; outcat[0].measure[Nmeas].averef = Nave; Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/calibrate.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/calibrate.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/calibrate.c (revision 40477) @@ -74,6 +74,6 @@ /* check if this reference code is an average magnitude */ if (Nsec != -1) { - CalM0 = secfilt[Nsec].M; - dCalM = secfilt[Nsec].dM; + CalM0 = secfilt[Nsec].MpsfChp; + dCalM = secfilt[Nsec].dMpsfChp; found0 = TRUE; } @@ -82,5 +82,5 @@ Nsec = GetPhotcodeNsec (CalC1); if (Nsec != -1) { - CalM1 = secfilt[Nsec].M; + CalM1 = secfilt[Nsec].MpsfChp; found1 = TRUE; } @@ -89,5 +89,5 @@ Nsec = GetPhotcodeNsec (CalC2); if (Nsec != -1) { - CalM2 = secfilt[Nsec].M; + CalM2 = secfilt[Nsec].MpsfChp; found2 = TRUE; } @@ -151,6 +151,7 @@ if (MaxN == -1) { fprintf (stderr, "no clean stars\n"); - image[0].Mcal = 10.000; - image[0].dMcal = 10.000; + image[0].McalPSF = 10.000; + image[0].McalAPER = 10.000; + image[0].dMcal = 10.000; return; } @@ -187,6 +188,7 @@ if (Nkeep < 5) { fprintf (stderr, "too few stars\n"); - image[0].Mcal = 10.000; - image[0].dMcal = 10.000; + image[0].McalPSF = 10.000; + image[0].McalAPER = 10.000; + image[0].dMcal = 10.000; return; } @@ -225,11 +227,13 @@ fprintf (stdout, "ZERO_POINT_NSTARS = %4.0f\n", N); // fprintf (stderr, "N: %.0f, mean: %f, wt mean: %f, stdev: %f, precision: %f\n", N, M1, Mw, M2, M2 / sqrt (N)); - image[0].Mcal = M1; - image[0].dMcal = M2 / sqrt (N); + image[0].McalPSF = M1; + image[0].McalAPER = M1; + image[0].dMcal = M2 / sqrt (N); image[0].nFitPhotom = N; } else { fprintf (stderr, "too few stars\n"); - image[0].Mcal = 10.000; - image[0].dMcal = 10.000; + image[0].McalPSF = 10.000; + image[0].McalAPER = 10.000; + image[0].dMcal = 10.000; image[0].nFitPhotom = 0; } Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/fakeimage.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/fakeimage.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/fakeimage.c (revision 40477) @@ -110,6 +110,8 @@ image[i+1].ccdnum = 0xff; - image[i+1].Mcal = 0.0; - image[i+1].Xm = NAN_S_SHORT; + image[i+1].McalPSF = 0.0; + image[i+1].McalAPER = 0.0; + image[i+1].McalChiSq = NAN; + image[i+1].dMcal = NAN; image[i+1].flags = 0; @@ -165,7 +167,11 @@ image[0].secz = 1.0; image[0].ccdnum = 0xff; - image[0].Mcal = 0.0; - image[0].Xm = NAN_S_SHORT; + + image[0].McalPSF = 0.0; + image[0].McalAPER = 0.0; + image[0].McalChiSq = NAN; + image[0].dMcal = NAN; image[0].flags = 0; + image[0].nstar = 0; Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/find_matches.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/find_matches.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/find_matches.c (revision 40477) @@ -212,6 +212,6 @@ /* in UPDATE mode, this value is not saved; use relphot to recalculate */ if (Nsec > -1) { - if (isnan(tgtcat[0].secfilt[n*Nsecfilt+Nsec].M)) { - tgtcat[0].secfilt[n*Nsecfilt+Nsec].M = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); + if (isnan(tgtcat[0].secfilt[n*Nsecfilt+Nsec].MpsfChp)) { + tgtcat[0].secfilt[n*Nsecfilt+Nsec].MpsfChp = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); } } @@ -305,5 +305,5 @@ /* in UPDATE mode, this value is not saved; use relphot to recalculate */ if (Nsec > -1) { - tgtcat[0].secfilt[Nave*Nsecfilt+Nsec].M = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); + tgtcat[0].secfilt[Nave*Nsecfilt+Nsec].MpsfChp = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); } Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/find_matches_closest.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/find_matches_closest.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/find_matches_closest.c (revision 40477) @@ -248,6 +248,6 @@ /* in UPDATE mode, this value is not saved; use relphot to recalculate */ if (Nsec > -1) { - if (isnan(tgtcat[0].secfilt[n*Nsecfilt+Nsec].M)) { - tgtcat[0].secfilt[n*Nsecfilt+Nsec].M = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); + if (isnan(tgtcat[0].secfilt[n*Nsecfilt+Nsec].MpsfChp)) { + tgtcat[0].secfilt[n*Nsecfilt+Nsec].MpsfChp = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); } } @@ -352,5 +352,5 @@ /* in UPDATE mode, this value is not saved; use relphot to recalculate */ if (Nsec > -1) { - tgtcat[0].secfilt[Nave*Nsecfilt+Nsec].M = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); + tgtcat[0].secfilt[Nave*Nsecfilt+Nsec].MpsfChp = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); } Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/find_matches_gaia.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/find_matches_gaia.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/find_matches_gaia.c (revision 40477) @@ -267,5 +267,5 @@ /* in UPDATE mode, this value is not saved; use relphot to recalculate */ if (Nsec > -1) { - tgtcat[0].secfilt[Nave*Nsecfilt+Nsec].M = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); + tgtcat[0].secfilt[Nave*Nsecfilt+Nsec].MpsfChp = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); } Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/find_proper.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/find_proper.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/find_proper.c (revision 40477) @@ -104,19 +104,21 @@ /** add measurements for this star **/ - catalog[0].measure[Nmeas].dR = 360000.0*(catalog[0].average[n].R - usno[N].R); - catalog[0].measure[Nmeas].dD = 360000.0*(catalog[0].average[n].D - usno[N].D); - catalog[0].measure[Nmeas].M = 1000.0*fabs(usno[N].r); - catalog[0].measure[Nmeas].Mcal= 0; /* above measurement is exact */ - catalog[0].measure[Nmeas].dM = 100; /* error in input files stored in thousandths of mag */ - catalog[0].measure[Nmeas].t = 0; /* a flag: if 0, image is not in database */ - catalog[0].measure[Nmeas].averef = n; - catalog[0].measure[Nmeas].photcode = USNO_RED; - catalog[0].measure[Nmeas+1].dR = catalog[0].measure[Nmeas].dR; - catalog[0].measure[Nmeas+1].dD = catalog[0].measure[Nmeas].dD; - catalog[0].measure[Nmeas+1].M = 1000.0*fabs(usno[N].b); - catalog[0].measure[Nmeas+1].Mcal= 0; /* above measurement is exact */ - catalog[0].measure[Nmeas+1].dM = 100; /* error in input files stored in thousandths of mag */ - catalog[0].measure[Nmeas+1].t = 0; /* a flag: if 0, image is not in database */ - catalog[0].measure[Nmeas+1].averef = n; + catalog[0].measure[Nmeas].dR = 360000.0*(catalog[0].average[n].R - usno[N].R); + catalog[0].measure[Nmeas].dD = 360000.0*(catalog[0].average[n].D - usno[N].D); + catalog[0].measure[Nmeas].M = 1000.0*fabs(usno[N].r); + catalog[0].measure[Nmeas].McalPSF = 0; /* above measurement is exact */ + catalog[0].measure[Nmeas].McalAPER = 0; /* above measurement is exact */ + catalog[0].measure[Nmeas].dM = 100; /* error in input files stored in thousandths of mag */ + catalog[0].measure[Nmeas].t = 0; /* a flag: if 0, image is not in database */ + catalog[0].measure[Nmeas].averef = n; + catalog[0].measure[Nmeas].photcode = USNO_RED; + catalog[0].measure[Nmeas+1].dR = catalog[0].measure[Nmeas].dR; + catalog[0].measure[Nmeas+1].dD = catalog[0].measure[Nmeas].dD; + catalog[0].measure[Nmeas+1].M = 1000.0*fabs(usno[N].b); + catalog[0].measure[Nmeas+1].McalPSF = 0; /* above measurement is exact */ + catalog[0].measure[Nmeas+1].McalAPER = 0; /* above measurement is exact */ + catalog[0].measure[Nmeas+1].dM = 100; /* error in input files stored in thousandths of mag */ + catalog[0].measure[Nmeas+1].t = 0; /* a flag: if 0, image is not in database */ + catalog[0].measure[Nmeas+1].averef = n; catalog[0].measure[Nmeas+1].photcode = USNO_BLUE; /* add flag in average to mark as matched with the USNO catalog */ Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/loadgalphot_readstars.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/loadgalphot_readstars.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/loadgalphot_readstars.c (revision 40477) @@ -241,5 +241,9 @@ int ifit = idx_gal[i]; - if (ifit < 0) { Nbad ++; continue; } // skip galaxy models with bad IDs + if (ifit < 0) { + fprintf (stderr, "%d %f %f %d\n", i, X_FIT[i], Y_FIT[i], MODEL_TYPE_gal[i]); + Nbad ++; + continue; + } // skip galaxy models with bad IDs double R, D; Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/loadsupercos_plates.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/loadsupercos_plates.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/loadsupercos_plates.c (revision 40477) @@ -72,7 +72,8 @@ image[Nimage].apmifit = 0.0; image[Nimage].dapmifit = 0.0; - image[Nimage].Mcal = 0.0; - image[Nimage].dMcal = 0.0; - image[Nimage].Xm = 0.0; + image[Nimage].McalPSF = 0.0; + image[Nimage].McalAPER = 0.0; + image[Nimage].McalChiSq = 0.0; + image[Nimage].dMcal = 0.0; char photname[128]; Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/loadsupercos_rawdata.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/loadsupercos_rawdata.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/loadsupercos_rawdata.c (revision 40477) @@ -86,5 +86,6 @@ // XXX fix these - newcat->measure[i].Mcal = 0.0; + newcat->measure[i].McalPSF = 0.0; + newcat->measure[i].McalAPER = 0.0; newcat->measure[i].dt = image[Ni].exptime; Index: /branches/czw_branch/20170908/Ohana/src/addstar/src/mkcmf.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/addstar/src/mkcmf.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/addstar/src/mkcmf.c (revision 40477) @@ -33,6 +33,9 @@ float BAD_PSFQF_FRAC = 0.0; +static Coords coords; static float exptime = 1.0; -static Coords coords; +static float aper_offset = 0.0; +static float aper_scale = 1.0; // this is set to 10^-0.4*aper_offset below is aper_offset is non-zero +static float kron_scale = 1.0; // extra offset between psf and kron mags static char reserved[] = "Reserved space. This line can be used to add a new FITS card."; @@ -178,4 +181,18 @@ } + // aperture-like and psf-like mags may have different effective zero points. + // by setting this, we can check that we can apply different zero points + if ((N = get_argument (argc, argv, "-aper-offset"))) { + remove_argument (N, &argc, argv); + aper_offset = atof (argv[N]); + aper_scale = pow(10.0, -0.4*aper_offset); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-kron-scale"))) { + remove_argument (N, &argc, argv); + kron_scale = atof (argv[N]); + remove_argument (N, &argc, argv); + } + // XXX note that the airmass and ra,dec,mjd can be inconsistent (for a given observatory location) float airmass = 1.0; @@ -254,5 +271,6 @@ struct timeval now; gettimeofday (&now, NULL); - long A = now.tv_sec + now.tv_usec * 1000000; + int B = getpid(); + long A = now.tv_sec - now.tv_usec * 1000000 + B*100000; srand48(A); } @@ -412,9 +430,12 @@ gfits_modify (&header, "NINPUTS", "%d", 1, 5); + // if we have multiple stacks using the same input image IDs, we run into trouble + int nFrame = 800 * drand48(); + int i; for (i = 0; i < 5; i++) { char field[64], expname[64]; sprintf (field, "INP_%04d", i); - sprintf (expname, "o5745g01%02do.356%03d.wrp.1199763.skycell.1315.090.fits", i, i); + sprintf (expname, "o5745g01%02do.%03d%03d.wrp.1199763.skycell.1315.090.fits", i, nFrame, i); gfits_modify (&header, field, "%s", 1, expname); @@ -791,5 +812,5 @@ stars[i].Y = Y[i]; stars[i].M = M[i]; - stars[i].Map = M[i] - 0.05; + stars[i].Map = M[i] + aper_offset - 0.05; // randomly give poor PSFQF values @@ -818,6 +839,6 @@ stars[i].flags = Flag[i]; - stars[i].kronFlux = flux * 1.25; - stars[i].kronFluxErr = fSN * flux * 1.25; + stars[i].kronFlux = flux * kron_scale * aper_scale; + stars[i].kronFluxErr = fSN * flux * kron_scale * aper_scale; } @@ -852,6 +873,6 @@ stars[i].Y = Y[i]; stars[i].M = M[i]; - stars[i].Map = M[i] - 0.05; - stars[i].MapRaw = M[i] - 0.10; + stars[i].Map = M[i] + aper_offset - 0.05; + stars[i].MapRaw = M[i] + aper_offset - 0.10; stars[i].Flux = flux; @@ -888,6 +909,6 @@ stars[i].flags = Flag[i]; - stars[i].kronFlux = flux * 1.25; - stars[i].kronFluxErr = fSN * flux * 1.25; + stars[i].kronFlux = flux * kron_scale * aper_scale; + stars[i].kronFluxErr = fSN * flux * kron_scale * aper_scale; } @@ -933,6 +954,6 @@ stars[i].dFlux = flux * fSN; - stars[i].Map = M[i] - 0.05; - stars[i].MapRaw = M[i] - 0.10; + stars[i].Map = M[i] + aper_offset - 0.05; + stars[i].MapRaw = M[i] + aper_offset - 0.10; stars[i].apRadius = 8.0; @@ -983,9 +1004,9 @@ stars[i].Mrh = FX; - stars[i].kronFlux = flux * 1.25; - stars[i].kronFluxErr = fSN * flux * 1.25; - - stars[i].kronInner = fSN * flux * 0.9; - stars[i].kronOuter = fSN * flux * 1.5; + stars[i].kronFlux = flux * kron_scale * aper_scale; + stars[i].kronFluxErr = fSN * flux * kron_scale * aper_scale; + + stars[i].kronInner = fSN * flux * 0.9 * aper_scale; + stars[i].kronOuter = fSN * flux * 1.5 * aper_scale; stars[i].skyLimitRad = 1; @@ -1040,6 +1061,6 @@ stars[i].dFlux = flux * fSN; - stars[i].Map = M[i] - 0.05; - stars[i].MapRaw = M[i] - 0.10; + stars[i].Map = M[i] + aper_offset - 0.05; + stars[i].MapRaw = M[i] + aper_offset - 0.10; stars[i].apRadius = 8.0; @@ -1090,9 +1111,9 @@ stars[i].Mrh = FX; - stars[i].kronFlux = flux * 1.25; - stars[i].kronFluxErr = fSN * flux * 1.25; - - stars[i].kronInner = fSN * flux * 0.9; - stars[i].kronOuter = fSN * flux * 1.5; + stars[i].kronFlux = flux * kron_scale * aper_scale; + stars[i].kronFluxErr = fSN * flux * kron_scale * aper_scale; + + stars[i].kronInner = fSN * flux * 0.9 * aper_scale; + stars[i].kronOuter = fSN * flux * 1.5 * aper_scale; // stars[i].skyLimitRad = 1; @@ -1144,6 +1165,6 @@ stars[i].dFlux = flux * fSN; - stars[i].Map = M[i] - 0.05; - stars[i].MapRaw = M[i] - 0.10; + stars[i].Map = M[i] + aper_offset - 0.05; + stars[i].MapRaw = M[i] + aper_offset - 0.10; stars[i].apRadius = 8.0; @@ -1185,9 +1206,9 @@ stars[i].Mrh = FX; - stars[i].kronFlux = flux * 1.25; - stars[i].kronFluxErr = fSN * flux * 1.25; - - stars[i].kronInner = fSN * flux * 0.9; - stars[i].kronOuter = fSN * flux * 1.5; + stars[i].kronFlux = flux * kron_scale * aper_scale; + stars[i].kronFluxErr = fSN * flux * kron_scale * aper_scale; + + stars[i].kronInner = fSN * flux * 0.9 * aper_scale; + stars[i].kronOuter = fSN * flux * 1.5 * aper_scale; stars[i].chipNum = 1; @@ -1270,6 +1291,6 @@ stars[i].dFlux = flux * fSN; - stars[i].Map = M[i] - 0.05; - stars[i].MapRaw = M[i] - 0.10; + stars[i].Map = M[i] + aper_offset - 0.05; + stars[i].MapRaw = M[i] + aper_offset - 0.10; stars[i].apRadius = 8.0; @@ -1347,9 +1368,9 @@ stars[i].srcChipY = Y[i] + 10.0; - stars[i].kronFlux = flux * 1.25; - stars[i].kronFluxErr = fSN * flux * 1.25; - - stars[i].kronInner = fSN * flux * 0.9; - stars[i].kronOuter = fSN * flux * 1.5; + stars[i].kronFlux = flux * kron_scale * aper_scale; + stars[i].kronFluxErr = fSN * flux * kron_scale * aper_scale; + + stars[i].kronInner = fSN * flux * 0.9 * aper_scale; + stars[i].kronOuter = fSN * flux * 1.5 * aper_scale; stars[i].skyLimitRad = 1; Index: /branches/czw_branch/20170908/Ohana/src/checkastro/src/BrightCatalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/checkastro/src/BrightCatalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/checkastro/src/BrightCatalog.c (revision 40477) @@ -71,5 +71,6 @@ GET_COLUMN(dD, "DEC_OFF", float); GET_COLUMN(M, "MAG_SYS", float); - GET_COLUMN(Mcal, "MAG_CAL", float); + GET_COLUMN(McalPSF, "MCAL_PSF", float); + GET_COLUMN(McalAPER, "MCAL_APER", float); GET_COLUMN(dM, "MAG_ERR", float); GET_COLUMN(airmass, "AIRMASS", float); @@ -95,5 +96,6 @@ measure[i].dD = dD[i]; measure[i].M = M[i]; - measure[i].Mcal = Mcal[i]; + measure[i].McalPSF = McalPSF[i]; + measure[i].McalAPER = McalAPER[i]; measure[i].dM = dM[i]; measure[i].airmass = airmass[i]; @@ -116,5 +118,6 @@ free (dD ); free (M ); - free (Mcal ); + free (McalPSF ); + free (McalAPER); free (dM ); free (airmass ); @@ -270,12 +273,12 @@ ALLOCATE (secfilt, SecFilt, Nrow); for (i = 0; i < Nrow; i++) { - secfilt[i].M = M[i]; - secfilt[i].dM = dM[i]; - secfilt[i].Mchisq= Mchisq[i]; - secfilt[i].flags = flags[i]; - secfilt[i].Ncode = Ncode[i]; - secfilt[i].Nused = Nused[i]; - secfilt[i].M_20 = M_20[i]; - secfilt[i].M_80 = M_80[i]; + secfilt[i].MpsfChp = M[i]; + secfilt[i].dMpsfChp = dM[i]; + secfilt[i].Mchisq = Mchisq[i]; + secfilt[i].flags = flags[i]; + secfilt[i].Ncode = Ncode[i]; + secfilt[i].Nused = Nused[i]; + secfilt[i].M_20 = M_20[i]; + secfilt[i].M_80 = M_80[i]; } fprintf (stderr, "loaded data for %lld averages\n", (long long) Nrow); @@ -366,5 +369,6 @@ float *dD ; ALLOCATE (dD , float, catalog->Nmeasure); float *M ; ALLOCATE (M , float, catalog->Nmeasure); - float *Mcal ; ALLOCATE (Mcal , float, catalog->Nmeasure); + float *McalPSF ; ALLOCATE (McalPSF , float, catalog->Nmeasure); + float *McalAPER ; ALLOCATE (McalAPER , float, catalog->Nmeasure); float *dM ; ALLOCATE (dM , float, catalog->Nmeasure); float *airmass ; ALLOCATE (airmass , float, catalog->Nmeasure); @@ -388,5 +392,6 @@ dD[i] = measure[i].dD ; M[i] = measure[i].M ; - Mcal[i] = measure[i].Mcal ; + McalPSF[i] = measure[i].McalPSF ; + McalAPER[i] = measure[i].McalAPER ; dM[i] = measure[i].dM ; airmass[i] = measure[i].airmass ; @@ -409,5 +414,6 @@ gfits_set_bintable_column (&theader, &ftable, "DEC_OFF", dD, catalog->Nmeasure); gfits_set_bintable_column (&theader, &ftable, "MAG_SYS", M, catalog->Nmeasure); - gfits_set_bintable_column (&theader, &ftable, "MAG_CAL", Mcal, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MCAL_PSF", McalPSF, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MCAL_APER", McalAPER, catalog->Nmeasure); gfits_set_bintable_column (&theader, &ftable, "MAG_ERR", dM, catalog->Nmeasure); gfits_set_bintable_column (&theader, &ftable, "AIRMASS", airmass, catalog->Nmeasure); @@ -428,5 +434,6 @@ free (dD ); free (M ); - free (Mcal ); + free (McalPSF ); + free (McalAPER); free (dM ); free (airmass ); @@ -652,12 +659,12 @@ SecFilt *secfilt = catalog->secfilt; for (i = 0; i < Nsec; i++) { - M [i] = secfilt[i].M ; - dM [i] = secfilt[i].dM ; - Mchisq[i] = secfilt[i].Mchisq ; - flags [i] = secfilt[i].flags ; - Ncode [i] = secfilt[i].Ncode ; - Nused [i] = secfilt[i].Nused ; - M_20 [i] = secfilt[i].M_20 ; - M_80 [i] = secfilt[i].M_80 ; + M [i] = secfilt[i].MpsfChp ; + dM [i] = secfilt[i].dMpsfChp; + Mchisq[i] = secfilt[i].Mchisq ; + flags [i] = secfilt[i].flags ; + Ncode [i] = secfilt[i].Ncode ; + Nused [i] = secfilt[i].Nused ; + M_20 [i] = secfilt[i].M_20 ; + M_80 [i] = secfilt[i].M_80 ; } Index: /branches/czw_branch/20170908/Ohana/src/delstar/Makefile =================================================================== --- /branches/czw_branch/20170908/Ohana/src/delstar/Makefile (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/delstar/Makefile (revision 40477) @@ -32,4 +32,5 @@ $(SRC)/delete_duplicate_measures.$(ARCH).o \ $(SRC)/delete_measures_by_match.$(ARCH).o \ +$(SRC)/delete_measures_by_detID.$(ARCH).o \ $(SRC)/delete_fix_LAP.$(ARCH).o \ $(SRC)/delete_fix_LAP_edges.$(ARCH).o \ @@ -58,4 +59,5 @@ $(SRC)/delete_duplicate_measures.$(ARCH).o \ $(SRC)/delete_measures_by_match.$(ARCH).o \ +$(SRC)/delete_measures_by_detID.$(ARCH).o \ $(SRC)/delete_fix_LAP.$(ARCH).o \ $(SRC)/delete_fix_LAP_edges.$(ARCH).o \ Index: /branches/czw_branch/20170908/Ohana/src/delstar/include/delstar.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/delstar/include/delstar.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/delstar/include/delstar.h (revision 40477) @@ -96,4 +96,5 @@ int SKIP_IMAGES; char *BACKUP_EXTNAME; +char *DELLIST_FILENAME; time_t START; @@ -105,5 +106,5 @@ int MODE; -enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES, MODE_DUP_MEASURES, MODE_DELETE_MEASURES_BY_MATCH, MODE_FIX_LAP, MODE_FIX_LAP_STATS, MODE_FIX_LAP_EDGES, MODE_FIX_LAP_EDGES_DELETE}; +enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES, MODE_DUP_MEASURES, MODE_DELETE_MEASURES_BY_MATCH, MODE_DELETE_MEASURES_BY_DETID, MODE_FIX_LAP, MODE_FIX_LAP_STATS, MODE_FIX_LAP_EDGES, MODE_FIX_LAP_EDGES_DELETE}; char DateKeyword[64], DateMode[64], UTKeyword[64], MJDKeyword[64], JDKeyword[64]; @@ -230,2 +231,19 @@ int delete_fix_LAP_edges_drop_measures (Catalog *catalog, MeasureEdge *measure_edge, off_t Nmeasure_edge, int catIDcount); int delete_fix_LAP_edges_delete_parallel (SkyList *sky); + +typedef struct { + int catID; + int *objID; + int *detID; + int *imageID; + int NdetID; +} CatIDGroup; + +int dvo_catalog_subset_backup (Catalog *catalog, char *suffix); +void isortthree (int *X, int *Y, int *Z, int N); +int thisCatID (int *catID, int NcatID, int index); +int needCatID (int *catID, int NcatID, int index); +int delete_measures_by_detID_catalog (Catalog *catalog, CatIDGroup *catIDgroup); +int delete_measures_by_detID_parallel (SkyList *sky); +int delete_measures_by_detID (); + Index: /branches/czw_branch/20170908/Ohana/src/delstar/src/args.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/delstar/src/args.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/delstar/src/args.c (revision 40477) @@ -202,4 +202,14 @@ } + DELLIST_FILENAME = NULL; + if ((N = get_argument (argc, argv, "-delete-measures-by-detID"))) { + if (MODE != MODE_NONE) usage(); + MODE = MODE_DELETE_MEASURES_BY_DETID; + remove_argument (N, &argc, argv); + SKIP_IMAGES = TRUE; // we do not need to load the images for -dup-measures + DELLIST_FILENAME = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + DELETE_MIN_DET_ID = 0; DELETE_MAX_DET_ID = 0; @@ -374,4 +384,5 @@ FREE (PHOTCODE_LIST); FREE (IMSTATS_FILE); + FREE (DELLIST_FILENAME); } @@ -494,4 +505,14 @@ remove_argument (N, &argc, argv); SKIP_IMAGES = TRUE; // we do not need to load the images for -dup-measures + } + + DELLIST_FILENAME = NULL; + if ((N = get_argument (argc, argv, "-delete-measures-by-detID"))) { + if (MODE != MODE_NONE) usage(); + MODE = MODE_DELETE_MEASURES_BY_DETID; + remove_argument (N, &argc, argv); + SKIP_IMAGES = TRUE; // we do not need to load the images for -dup-measures + DELLIST_FILENAME = strcreate (argv[N]); + remove_argument (N, &argc, argv); } @@ -603,4 +624,5 @@ FREE (MEASURE_EDGE_FILE); FREE (IMSTATS_FILE); + FREE (DELLIST_FILENAME); } Index: /branches/czw_branch/20170908/Ohana/src/delstar/src/delete_measures_by_detID.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/delstar/src/delete_measures_by_detID.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/delstar/src/delete_measures_by_detID.c (revision 40477) @@ -0,0 +1,667 @@ +# include "delstar.h" + +// this function identifies detections to be deleted by matching the supplied values (detID, imageID, objID, catID) + +/* Basic Outline + + load a list of detID imageID objID catID from a file + group by catID -> objID -> detID,imageID + + loop over catalogs, load desired catIDs + + load catalog, loop over objects + loop over detections, delete on imageID, detID + */ + +int delete_measures_by_detID () { + + int i, j; + Catalog catalog; + + // load the current sky table (layout of all SkyRegions) + SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, TRUE, -1, VERBOSE); + if (!sky) { + fprintf (stderr, "ERROR running loading sky table from %s\n", CATDIR); + exit (2); + } + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // determine the populated SkyRegions overlapping the requested area (default depth) + SkyList *skylist = NULL; + if (SINGLE_CPT) { + skylist = SkyRegionByCPT (sky, SINGLE_CPT); + } else { + skylist = SkyListByPatch (sky, -1, &UserPatch); + } + if (!skylist) { + fprintf (stderr, "ERROR setting up skylist for %s\n", CATDIR); + exit (2); + } + + // launch the remote jobs + if (PARALLEL && !HOST_ID) { + int status = delete_measures_by_detID_parallel (skylist); + + SkyTableFree (sky); + SkyListFree (skylist); + FreePhotcodeTable (); + + return status; + } + + // load the table of detections to delete + FILE *f = fopen (DELLIST_FILENAME, "r"); + if (f == NULL) { + fprintf (stderr, "ERROR cannot open ID file %s\n", DELLIST_FILENAME); + exit (2); + } + + int Ndelete = 0; + int NDELETE = 5000; + ALLOCATE_PTR (imageID, int, NDELETE); + ALLOCATE_PTR (detID, int, NDELETE); + ALLOCATE_PTR (catID, int, NDELETE); + ALLOCATE_PTR (objID, int, NDELETE); + + while (fscanf (f, "%*s %*s %d %d %d %d %*f", &imageID[Ndelete], &detID[Ndelete], &objID[Ndelete], &catID[Ndelete]) != EOF) { + Ndelete ++; + if (Ndelete >= NDELETE) { + NDELETE += 5000; + REALLOCATE (imageID, int, NDELETE); + REALLOCATE (detID, int, NDELETE); + REALLOCATE (catID, int, NDELETE); + REALLOCATE (objID, int, NDELETE); + } + } + + // generate a unique list of catIDs + ALLOCATE_PTR (catIDcopy, int, Ndelete); + memcpy (catIDcopy, catID, Ndelete*sizeof(int)); + isort (catIDcopy, Ndelete); + + // NcatID tracks the number of unique catID values + int NcatID = 0; + int NCATID = 500; + + ALLOCATE_PTR (catIDuniq, int, NCATID); + + for (i = 0; i < Ndelete; NcatID++) { + if (NcatID >= NCATID) { + NCATID += 10000; + REALLOCATE (catIDuniq, int, NCATID); + } + catIDuniq[NcatID] = catIDcopy[i]; + int lastValue = catIDcopy[i]; + while ((i < Ndelete) && (catIDcopy[i] == lastValue)) { + i++; + } + } + // fprintf (stderr, "found %d uniq catIDs\n", NcatID); + + ALLOCATE_PTR (catIDgroup, CatIDGroup, NcatID); + + // generate a set of lookup tables for each catID + for (i = 0; i < NcatID; i++) { + catIDgroup[i].catID = catIDuniq[i]; + + int NdetID = 0; + int NDETID = 1000; + ALLOCATE (catIDgroup[i].detID, int, NDETID); + ALLOCATE (catIDgroup[i].objID, int, NDETID); + ALLOCATE (catIDgroup[i].imageID, int, NDETID); + + for (j = 0; j < Ndelete; j++) { + if (catID[j] != catIDuniq[i]) continue; + + catIDgroup[i].detID[NdetID] = detID[j]; + catIDgroup[i].objID[NdetID] = objID[j]; + catIDgroup[i].imageID[NdetID] = imageID[j]; + NdetID ++; + if (NdetID >= NDETID) { + NDETID += 1000; + REALLOCATE (catIDgroup[i].detID, int, NDETID); + REALLOCATE (catIDgroup[i].objID, int, NDETID); + REALLOCATE (catIDgroup[i].imageID, int, NDETID); + } + } + catIDgroup[i].NdetID = NdetID; + isortthree (catIDgroup[i].objID, catIDgroup[i].detID, catIDgroup[i].imageID, catIDgroup[i].NdetID); + } + + // delete detections from region + for (i = 0; i < skylist[0].Nregions; i++) { + + if (VERBOSE) fprintf (stderr, "checking %s : %d, %d\n", skylist[0].regions[i]->name, skylist[0].regions[i]->index, skylist[0].regions[i]->hostID); + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue; + + // skip until we find a catalog we need + if (!needCatID (catIDuniq, NcatID, skylist[0].regions[i]->index)) continue; + + // set up the basic catalog info + char hostfile[1024]; + snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name); + + dvo_catalog_init (&catalog, TRUE); + catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i]; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT; + + if (VERBOSE) fprintf (stderr, "deleting from %s\n", catalog.filename); + + // an error exit status here is a significant error + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE2, "a")) { + fprintf (stderr, "ERROR: failure to open catalog file %s\n", catalog.filename); + exit (2); + } + if (!catalog.Naverage_disk) { + if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + int myCatID = thisCatID (catIDuniq, NcatID, skylist[0].regions[i]->index); + myAssert (myCatID >= 0, "error"); + + int Ndel_this_cat = delete_measures_by_detID_catalog (&catalog, &catIDgroup[myCatID]); + + if (!Ndel_this_cat) { + fprintf (stderr, "no changes to %s, no output\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + char history[128]; + struct timeval now; + gettimeofday (&now, (void *) NULL); + char *moddate = ohana_sec_to_date (now.tv_sec); + snprintf (history, 128, "delete %d measurements by detID: %s", Ndel_this_cat, moddate); + gfits_modify_alt (&catalog.header, "HISTORY", "%S", 0, history); // adds a new entry + free (moddate); + + // save backup of original cpm file + if (!dvo_catalog_subset_backup (&catalog, BACKUP_EXTNAME)) { + fprintf (stderr, "ERROR: failed to make backup cpt table for catalog %s\n", catalog.filename); + exit (1); + } + if (!dvo_catalog_subset_backup (catalog.measure_catalog, BACKUP_EXTNAME)) { + fprintf (stderr, "ERROR: failed to make backup cpm table for catalog %s\n", catalog.filename); + exit (1); + } + if (!dvo_catalog_subset_backup (catalog.secfilt_catalog, BACKUP_EXTNAME)) { + fprintf (stderr, "ERROR: failed to make backup cps table for catalog %s\n", catalog.filename); + exit (1); + } + + // XXX something of a hack : I only want to save average, measure, secfilt. + catalog.Nmissing = catalog.Nmissing_off; + catalog.Nlensing = catalog.Nlensing_off; + catalog.Nlensobj = catalog.Nlensobj_off; + catalog.Nstarpar = catalog.Nstarpar_off; + catalog.Ngalphot = catalog.Ngalphot_off; + + catalog.Nmissing_off = 0; + catalog.Nlensing_off = 0; + catalog.Nlensobj_off = 0; + catalog.Nstarpar_off = 0; + catalog.Ngalphot_off = 0; + + SetProtect (TRUE); + dvo_catalog_save_complete (&catalog, VERBOSE2); + dvo_catalog_unlock (&catalog); + SetProtect (FALSE); + + dvo_catalog_free (&catalog); + } + + SkyTableFree (sky); + SkyListFree (skylist); + FreePhotcodeTable (); + + free (imageID); + free (detID); + free (objID); + free (catID); + free (catIDcopy); + free (catIDuniq); + + for (i = 0; i < NcatID; i++) { + free (catIDgroup[i].objID); + free (catIDgroup[i].detID); + free (catIDgroup[i].imageID); + } + free (catIDgroup); + + return TRUE; +} + +// CATDIR is supplied globally +int delete_measures_by_detID_parallel (SkyList *sky) { + + // launch the delstar_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + exit (1); + } + + char *tmppath = abspath (DELLIST_FILENAME, DVO_MAX_PATH); + free (DELLIST_FILENAME); + DELLIST_FILENAME = tmppath; + + int i, j; + for (i = 0; i < table->Nhosts; i++) { + + if (sky->Nregions < table->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + char *command = NULL; + strextend (&command, "delstar_client -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -delete-measures-by-detID %s", + table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, + UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, DELLIST_FILENAME); + + if (VERBOSE) { strextend (&command, "-v"); } + if (VERBOSE2) { strextend (&command, "-vv"); } + if (SINGLE_CPT) { strextend (&command, "-cpt %s", SINGLE_CPT); } + if (UPDATE) { strextend (&command, "-update"); } + if (SAVE_DELETES) { strextend (&command, "-save-deletes"); } + + if (BACKUP_EXTNAME) { strextend (&command, "-backup-extname %s", BACKUP_EXTNAME); } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) { + free (command); + continue; + } + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running delstar_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + free (command); + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + } + + FreeHostTable(table); + + return TRUE; +} + +int delete_measures_by_detID_catalog (Catalog *catalog, CatIDGroup *catIDgroup) { + + // double-check the CATID in the cpt header? + + Measure *measureOut = NULL; + Average *averageOut = NULL; + SecFilt *secfiltOut = NULL; + + off_t *measureDrop, *measureSeqRaw, *measureSeqOut, *measureRefOut, *measureAveRaw, *averageNmeas, *averageDmeas, *averageSeqOut, *averageRefOut, *measureAveOut; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + /* internal counters */ + off_t Nmeasure = catalog[0].Nmeasure; + off_t Naverage = catalog[0].Naverage; + + Measure *measure = catalog[0].measure; + Average *average = catalog[0].average; + SecFilt *secfilt = catalog[0].secfilt; + + if (VERBOSE) fprintf (stderr, "starting with Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT"\n", catalog[0].Naverage, catalog[0].Nmeasure); + + // we have a table of average objects and an unsorted table of measurements. each measurement + // has a reference to the average object sequence (as well as an ID) + // measure[i].averef -> average[averef] + // measure[i].objID = average[averef].objID + // measure[i].catID = average[averef].catID + + // we want a sorted measure array with all averef entries in sequence, skipping the entries which match our criteria + + // arrays + ALLOCATE (measureDrop, off_t, Nmeasure); // mark measures to drop (0 == keep, 1 == drop) + ALLOCATE (measureSeqRaw, off_t, Nmeasure); // original sequence of measures (0 ... Nmeasure - 1) + ALLOCATE (measureAveRaw, off_t, Nmeasure); // averef values for original measure sequence (measure[N].averef : N = measureSeqRaw[i], averef = measureAveRaw[i] + + ALLOCATE (measureSeqOut, off_t, Nmeasure); // sequence of measures for output (resorted by averef and culled) + ALLOCATE (measureRefOut, off_t, Nmeasure); // reference to original sequence for new output sequence + ALLOCATE (averageNmeas, off_t, Naverage); // output number of measures for each average (average.Nmeasure) + ALLOCATE (averageDmeas, off_t, Naverage); // output starting point of measures for each average (average.measureOffset) + ALLOCATE (averageSeqOut, off_t, Naverage); // output sequence for averages + ALLOCATE (averageRefOut, off_t, Naverage); // raw references for output sequence + // NOTE since we no longer drop any averages even if they have no measures, these indexes are probably not needed + + memset (measureDrop, 0, sizeof(off_t) * Nmeasure); + + for (off_t i = 0; i < Naverage; i++) { + averageNmeas[i] = 0; + averageDmeas[i] = -1; + } + +# if (1) + FILE *fsave = NULL; + if (SAVE_DELETES) { + char savename[DVO_MAX_PATH]; + snprintf (savename, DVO_MAX_PATH, "%s.save.20171019.v1", catalog->filename); + struct stat filestat; + int myStatus = stat (savename, &filestat); + if (myStatus && (errno == ENOENT)) { + fsave = fopen (savename, "w"); + if (!fsave) { + fprintf (stderr, "problem opening file %s for output\n", savename); + } + } else { + if (!myStatus) { + fprintf (stderr, "file %s exists: will not overwrite\n", savename); + } + } + } +# endif + + // loop over average[i] to find the next objID + + int Ndrop = 0; + for (off_t i = 0, j = 0; (i < Naverage) && (j < catIDgroup->NdetID); ) { + + if (average[i].objID < catIDgroup->objID[j]) { i++; continue; } + if (average[i].objID > catIDgroup->objID[j]) { j++; continue; } + + // we have found an objID which we want to delete: + fprintf (stderr, "deleting from (catID, objID) = (%d, %d)\n", catIDgroup->catID, catIDgroup->objID[j]); + + int FirstObj = j; + + // look for all detections for this object which are in the list of desired detID, imageID pairs + off_t m = average[i].measureOffset; + for (int Nm = 0; Nm < average[i].Nmeasure; Nm++) { + for (j = FirstObj; (j < catIDgroup->NdetID) && (catIDgroup->objID[j] == average[i].objID); j++) { + myAssert (measure[Nm + m].objID == catIDgroup->objID[j], "oops"); + myAssert (measure[Nm + m].catID == catIDgroup->catID, "oops"); + if (measure[Nm + m].detID != catIDgroup->detID[j]) continue; + if (measure[Nm + m].imageID != catIDgroup->imageID[j]) continue; + // found one + measureDrop[Nm + m] = TRUE; + + off_t Nave = measure[Nm + m].averef; + if (VERBOSE) fprintf (stderr, "0x%08x 0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[Nm + m].imageID, measure[Nm + m].detID, measure[Nm + m].catID, average[Nave].R, average[Nave].D, measure[Nm + m].photcode); + if (fsave) { + fprintf (fsave, "0x%08x 0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[Nm + m].imageID, measure[Nm + m].detID, measure[Nm + m].catID, average[Nave].R, average[Nave].D, measure[Nm + m].photcode); + } + Ndrop ++; + } + } + + // the block above should find all detID,imageID matches for this objID. scan to the next objID in the catID group + j = FirstObj; + while ((j < catIDgroup->NdetID) && (average[i].objID == catIDgroup->objID[j])) j++; + } + + if (fsave) fclose (fsave); + + // set up the measure sequence lists + for (int i = 0; i < Nmeasure; i++) { + measureSeqRaw[i] = i; + measureAveRaw[i] = measure[i].averef; + } + + // sort measureSeqRaw and measureAveRaw in order of measureAveRaw + SortAveMeasMatch(measureSeqRaw, measureAveRaw, Nmeasure); + + // generate the measureSeqOut array + off_t NmeasOut = 0; + for (off_t i = 0; i < Nmeasure; i++) { + off_t j = measureSeqRaw[i]; + if (measureDrop[j]) continue; + measureSeqOut[NmeasOut] = j; + measureRefOut[j] = NmeasOut; + NmeasOut ++; + } + // n = measureSeqOut[i] : measureOut[i] = measure[n] (i = 0 -- NmeasOut, n = 0 -- Nmeasure) + // n = measureRefOut[i] : measureOut[n] = measure[i] + ALLOCATE (measureAveOut, off_t, NmeasOut); + + // count the number of measures for each averef + off_t N = 0; + off_t D = -1; + off_t currentAve = measureAveRaw[0]; + for (off_t i = 0; i < Nmeasure; i++) { + if (measureAveRaw[i] != currentAve) { + // we have hit the next entry in the list + averageNmeas[currentAve] = N; // number of measures + averageDmeas[currentAve] = D; // first measure + N = 0; + D = -1; + currentAve = measureAveRaw[i]; + } + off_t j = measureSeqRaw[i]; + if (measureDrop[j]) continue; + N++; + if (D == -1) { + // first valid measureOut for this averef + D = measureRefOut[j]; + } + } + averageNmeas[currentAve] = N; // number of measures + averageDmeas[currentAve] = D; // first measure + + // generate the new average sequence, skipping entries with no measurements + // update 20160524 : keep entries with no measurements + // NOTE: with the above update, averageSeqOut = averageRefOut = 0 ... Naverage - 1 + off_t NaveOut = 0; + for (off_t i = 0; i < Naverage; i++) { + // if (averageNmeas[i] == 0) continue; + averageSeqOut[NaveOut] = i; + averageRefOut[i] = NaveOut; + NaveOut ++; + } + // n = averageSeqOut[i] : averageOut[i] = average[n] (i = 0 -- NaveOut, n = 0 -- Naverage) + // n = averageRefOut[i] : averageOut[n] = average[i] + + // generate the output averefs + for (off_t i = 0; i < NmeasOut; i++) { + off_t j = measureSeqOut[i]; // measure[i] = measureOut[i] + off_t n = measureAveRaw[j]; // average[n] : measure[i] + off_t N = averageRefOut[n]; // averageOut[N] = average[n]; + measureAveOut[i] = N; // measureOut[i].averef = measureAveOut[i] + } + + // copy the (kept) measurements in the sorted order + ALLOCATE (measureOut, Measure, NmeasOut); + for (off_t i = 0; i < NmeasOut; i++) { + off_t j = measureSeqOut[i]; + measureOut[i] = measure[j]; + measureOut[i].averef = measureAveOut[i]; + } + + // copy the (kept) average entries + ALLOCATE (averageOut, Average, NaveOut); + for (off_t i = 0; i < NaveOut; i++) { + off_t j = averageSeqOut[i]; + averageOut[i] = average[j]; + averageOut[i].Nmeasure = averageNmeas[j]; + averageOut[i].measureOffset = averageDmeas[j]; + } + + // copy the secfilt entries for the (kept) average entries + ALLOCATE (secfiltOut, SecFilt, NaveOut*Nsecfilt); + for (off_t i = 0; i < NaveOut; i++) { + off_t j = averageSeqOut[i]; + for (off_t m = 0; m < Nsecfilt; m++) { + secfiltOut[i*Nsecfilt + m] = secfilt[j*Nsecfilt + m]; + } + } + + // update the values of average.measureOffset and average.Nmeasure + FREE(measure); + FREE(average); + FREE(secfilt); + catalog[0].measure = measureOut; + catalog[0].average = averageOut; + catalog[0].secfilt = secfiltOut; + catalog[0].Nmeasure = NmeasOut; + catalog[0].Naverage = NaveOut; + + // update catalog.Nmeasure and catalog.Naverage, double check + int NmeasureTotal = 0; + int measureOffsetOK = TRUE; + int averefOK = TRUE; + int objIDisOK = TRUE; + for (off_t i = 0; i < NaveOut; i++) { + NmeasureTotal += catalog[0].average[i].Nmeasure; + if (VERBOSE2 && !(NmeasureTotal <= catalog[0].Nmeasure)) { + fprintf (stderr, "too few measurements: %d %d %d\n", (int) i, NmeasureTotal, (int) catalog[0].Nmeasure); + } + measureOffsetOK &= (catalog[0].average[i].measureOffset < catalog[0].Nmeasure); + if (VERBOSE2 && !(catalog[0].average[i].measureOffset < catalog[0].Nmeasure)) { + fprintf (stderr, "offset too large: %d %d %d\n", (int) i, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure); + } + measureOffsetOK &= (catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure); + if (VERBOSE2 && !(catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure)) { + fprintf (stderr, "orrset + Nmeasure too large: %d + %d > %d %d\n", (int) i, catalog[0].average[i].measureOffset, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure); + } + off_t m = catalog[0].average[i].measureOffset; + for (off_t j = 0; j < catalog[0].average[i].Nmeasure; j++) { + averefOK &= (catalog[0].measure[m+j].averef == i); + if (VERBOSE2 && !(catalog[0].measure[m+j].averef == i)) { + fprintf (stderr, "averef broken: %d vs %d (measure %d)\n", (int) i, catalog[0].measure[m+j].averef, (int) (m+j)); + } + objIDisOK &= (catalog[0].measure[m+j].objID == catalog[0].average[i].objID); + if (VERBOSE2 && !(catalog[0].measure[m+j].objID == catalog[0].average[i].objID)) { + fprintf (stderr, "objID broken: %d vs %d (average %d, measure %d)\n", (int) i, (int) (m + j), catalog[0].average[i].objID, catalog[0].measure[m+j].objID); + } + } + } + + if (!measureOffsetOK) { + fprintf (stderr, "ERROR: catalog %s has an invalid measureOffset\n", catalog[0].filename); + } + + if (!averefOK) { + fprintf (stderr, "ERROR: catalog %s has invalid averefs\n", catalog[0].filename); + } + + if (NmeasureTotal != catalog[0].Nmeasure) { + fprintf (stderr, "ERROR: catalog %s has an invalid Nmeasure\n", catalog[0].filename); + } + + if (!objIDisOK) { + fprintf (stderr, "ERROR: catalog %s has invalid objIDs\n", catalog[0].filename); + } + + // MARKTIME(" match time %9.4f sec for %7lld measures, %6lld average\n", dtime, (long long) Nmeasure, (long long) Naverage); + + catalog[0].sorted = TRUE; + + if (VERBOSE) fprintf (stderr, "ending with Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT"\n", catalog[0].Naverage, catalog[0].Nmeasure); + + FREE (measureDrop); + FREE (measureSeqRaw); + FREE (measureSeqOut); + FREE (measureRefOut); + FREE (measureAveRaw); + FREE (averageNmeas); + FREE (averageDmeas); + FREE (averageSeqOut); + FREE (averageRefOut); + FREE (measureAveOut); + + return Ndrop; +} + +int needCatID (int *catID, int NcatID, int index) { + + // catID is a sorted list of desired catIDs + // search for index in the list + + // returns the sequence number of the last value < index + // itest is guaranteed to be < NcatID + int itest = ohana_bisection_int (catID, NcatID, index); + + if (catID[itest] == index) return TRUE; + + while ((itest < NcatID) && (catID[itest] < index)) itest ++; + if (itest >= NcatID) return FALSE; + if (catID[itest] == index) return TRUE; + return FALSE; +} + +int thisCatID (int *catID, int NcatID, int index) { + + // catID is a sorted list of desired catIDs + // search for index in the list + + // returns the sequence number of the last value < index + // itest is guaranteed to be < NcatID + int itest = ohana_bisection_int (catID, NcatID, index); + + if (catID[itest] == index) return itest; + + while ((itest < NcatID) && (catID[itest] < index)) itest ++; + if (itest >= NcatID) return -1; + if (catID[itest] == index) return itest; + return -1; +} + +void isortthree (int *X, int *Y, int *Z, int N) { + +# define SWAPFUNC(A,B){ int tmp; \ + tmp = X[A]; X[A] = X[B]; X[B] = tmp; \ + tmp = Y[A]; Y[A] = Y[B]; Y[B] = tmp; \ + tmp = Z[A]; Z[A] = Z[B]; Z[B] = tmp; \ +} +# define COMPARE(A,B)(X[A] < X[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + Index: /branches/czw_branch/20170908/Ohana/src/delstar/src/delstar.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/delstar/src/delstar.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/delstar/src/delstar.c (revision 40477) @@ -34,4 +34,11 @@ case MODE_DELETE_MEASURES_BY_MATCH: if (!delete_measures_by_match ()) exit (1); + delstar_args_free (); + ohana_memcheck (TRUE); + ohana_memdump (TRUE); + exit (0); + break; + case MODE_DELETE_MEASURES_BY_DETID: + if (!delete_measures_by_detID ()) exit (1); delstar_args_free (); ohana_memcheck (TRUE); Index: /branches/czw_branch/20170908/Ohana/src/delstar/src/delstar_client.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/delstar/src/delstar_client.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/delstar/src/delstar_client.c (revision 40477) @@ -37,4 +37,11 @@ case MODE_DELETE_MEASURES_BY_MATCH: if (!delete_measures_by_match ()) exit (1); + delstar_args_free (); + ohana_memcheck (TRUE); + ohana_memdump (TRUE); + exit (0); + break; + case MODE_DELETE_MEASURES_BY_DETID: + if (!delete_measures_by_detID ()) exit (1); delstar_args_free (); ohana_memcheck (TRUE); Index: /branches/czw_branch/20170908/Ohana/src/delstar/src/gimages.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/delstar/src/gimages.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/delstar/src/gimages.c (revision 40477) @@ -88,6 +88,7 @@ /* secz is in units milli-airmass */ - image[0].Mcal = ALPHA*(image[0].secz - 1.000); - image[0].Xm = NAN_S_SHORT; + image[0].McalPSF = ALPHA*(image[0].secz - 1.000); + image[0].McalAPER = ALPHA*(image[0].secz - 1.000); + image[0].McalChiSq = NAN_S_SHORT; free (header.buffer); Index: /branches/czw_branch/20170908/Ohana/src/dvolens/src/update_objects_catalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvolens/src/update_objects_catalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvolens/src/update_objects_catalog.c (revision 40477) @@ -219,5 +219,6 @@ // relphot sets measure->Mcal (setMcalOutput.c, called by setMrelFinal.c) - float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - measure->Mcal; + // XXX : I'm using McalAPER since these lens measurements are aperture-like, right? + float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - measure->McalAPER; float Fcal = 3630.8 * MagToFlux(Mcal); Index: /branches/czw_branch/20170908/Ohana/src/dvomerge/include/dvomerge.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvomerge/include/dvomerge.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvomerge/include/dvomerge.h (revision 40477) @@ -55,4 +55,6 @@ int MATCH_BY_EXTERN_ID; +int ONLY_MATCHES; + int MATCHED_TABLES; int RESET_STARPAR; Index: /branches/czw_branch/20170908/Ohana/src/dvomerge/src/args.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvomerge/src/args.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvomerge/src/args.c (revision 40477) @@ -86,8 +86,14 @@ } - /* extra error messages */ + /* match images in the src and tgt database tables by their externID values (otherwise by time and photcode) */ MATCH_BY_EXTERN_ID = FALSE; if ((N = get_argument (*argc, argv, "-match-by-extern-id"))) { MATCH_BY_EXTERN_ID = TRUE; + remove_argument (N, argc, argv); + } + /* add objects from input to output database only if they match an existing object */ + ONLY_MATCHES = FALSE; + if ((N = get_argument (*argc, argv, "-only-matches"))) { + ONLY_MATCHES = TRUE; remove_argument (N, argc, argv); } @@ -373,8 +379,14 @@ } - /* extra error messages */ + /* match images in the src and tgt database tables by their externID values (otherwise by time and photcode) */ MATCH_BY_EXTERN_ID = FALSE; if ((N = get_argument (*argc, argv, "-match-by-extern-id"))) { MATCH_BY_EXTERN_ID = TRUE; + remove_argument (N, argc, argv); + } + /* add objects from input to output database only if they match an existing object */ + ONLY_MATCHES = FALSE; + if ((N = get_argument (*argc, argv, "-only-matches"))) { + ONLY_MATCHES = TRUE; remove_argument (N, argc, argv); } Index: /branches/czw_branch/20170908/Ohana/src/dvomerge/src/build_links.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvomerge/src/build_links.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvomerge/src/build_links.c (revision 40477) @@ -701,5 +701,5 @@ off_t *next_galphot; - if (galphot) return NULL; + if (!galphot) return NULL; if (SKIP_GALPHOT) return NULL; Index: /branches/czw_branch/20170908/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 40477) @@ -330,4 +330,5 @@ if (MATCHED_TABLES) { strextend (&command, "-matched-tables"); } if (MATCH_BY_EXTERN_ID) { strextend (&command, "-match-by-extern-id"); } + if (ONLY_MATCHES) { strextend (&command, "-only-matches"); } if (UPDATE_CATFORMAT) { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); } if (UPDATE_CATCOMPRESS) { strextend (&command, "-update-catcompress %s", UPDATE_CATCOMPRESS); } Index: /branches/czw_branch/20170908/Ohana/src/dvomerge/src/dvosecfilt_catalogs.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvomerge/src/dvosecfilt_catalogs.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvomerge/src/dvosecfilt_catalogs.c (revision 40477) @@ -41,5 +41,5 @@ // always load all of the data (if any exists) catalog.Nsecfilt = 0; - catalog.catflags = DVO_LOAD_SECFILT; // XXX this will fail for MEF version + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; // XXX this will fail for MEF version catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data Index: /branches/czw_branch/20170908/Ohana/src/dvomerge/src/merge_catalogs_old.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 40477) @@ -367,6 +367,6 @@ // secfilt tables of input and output are the same for (k = 0; k < NsecfiltIn; k++) { - if ( isfinite(output[0].secfilt[n*NsecfiltIn+k].M)) continue; - if (!isfinite( input[0].secfilt[N*NsecfiltIn+k].M)) continue; + if ( isfinite(output[0].secfilt[n*NsecfiltIn+k].MpsfChp)) continue; + if (!isfinite( input[0].secfilt[N*NsecfiltIn+k].MpsfChp)) continue; output[0].secfilt[n*NsecfiltIn+k] = input[0].secfilt[N*NsecfiltIn+k]; } @@ -378,6 +378,6 @@ int outputIndex = n * NsecfiltOut + secfiltMap[k]; - if ( isfinite(output[0].secfilt[outputIndex].M)) continue; - if (!isfinite( input[0].secfilt[N*NsecfiltIn+k].M)) continue; + if ( isfinite(output[0].secfilt[outputIndex].MpsfChp)) continue; + if (!isfinite( input[0].secfilt[N*NsecfiltIn+k].MpsfChp)) continue; output[0].secfilt[outputIndex] = input[0].secfilt[N*NsecfiltIn+k]; } @@ -406,5 +406,5 @@ /** incorporate unmatched image stars, if this star is in field of this catalog **/ /* these new entries are all written out in UPDATE mode */ - for (i = 0; i < Nstars; i++) { + for (i = 0; (i < Nstars) && !ONLY_MATCHES; i++) { off_t N = N1[i]; @@ -489,5 +489,5 @@ } - if (isfinite(input[0].secfilt[N*NsecfiltIn+j].M)) { + if (isfinite(input[0].secfilt[N*NsecfiltIn+j].MpsfChp)) { output[0].secfilt[outputIndex] = input[0].secfilt[N*NsecfiltIn+j]; } Index: /branches/czw_branch/20170908/Ohana/src/dvopsps/include/dvopsps.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvopsps/include/dvopsps.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvopsps/include/dvopsps.h (revision 40477) @@ -19,6 +19,8 @@ float raErr; float decErr; - float zp; - float zpFactor; + float zpPSF; + float zpFactorPSF; + float zpAPER; + float zpFactorAPER; float telluricExt; float airmass; @@ -34,4 +36,5 @@ unsigned int flags; unsigned int objflags; + unsigned int filtflags; } Detections; @@ -45,4 +48,5 @@ int SAVE_REMOTE; +int TEST_MODE; int PARALLEL; @@ -96,5 +100,5 @@ int insert_detections_mysql_array PROTO((MYSQL *mysql, Detections *detections, int Ndetections)); int insert_detections_mysql_detvalue PROTO((IOBuffer *buffer, Detections *detection)); -int assign_detection_values PROTO((Detections *detection, Measure *measure, Average *average)); +int assign_detection_values PROTO((Detections *detection, Measure *measure, Average *average, SecFilt *secfilt)); int init_detections PROTO((void)); Index: /branches/czw_branch/20170908/Ohana/src/dvopsps/src/DetectionOps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvopsps/src/DetectionOps.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvopsps/src/DetectionOps.c (revision 40477) @@ -56,6 +56,8 @@ GET_COLUMN(raErr , "raErr", float); GET_COLUMN(decErr , "decErr", float); - GET_COLUMN(zp , "zp", float); - GET_COLUMN(zpFactor , "zpFactor", float); + GET_COLUMN(zpPSF , "zpPSF", float); + GET_COLUMN(zpFactorPSF , "zpFactorPSF", float); + GET_COLUMN(zpAPER , "zpAPER", float); + GET_COLUMN(zpFactorAPER, "zpFactorAPER",float); GET_COLUMN(telluricExt , "telluricExt", float); GET_COLUMN(airmass , "airmass", float); @@ -84,6 +86,8 @@ detections[i].raErr = raErr[i]; detections[i].decErr = decErr[i]; - detections[i].zp = zp[i]; - detections[i].zpFactor = zpFactor[i]; + detections[i].zpPSF = zpPSF[i]; + detections[i].zpFactorPSF = zpFactorPSF[i]; + detections[i].zpAPER = zpAPER[i]; + detections[i].zpFactorAPER = zpFactorAPER[i]; detections[i].telluricExt = telluricExt[i]; detections[i].airmass = airmass[i]; @@ -110,6 +114,8 @@ free (raErr ); free (decErr ); - free (zp ); - free (zpFactor ); + free (zpPSF ); + free (zpFactorPSF); + free (zpAPER ); + free (zpFactorAPER); free (telluricExt); free (airmass ); @@ -180,6 +186,8 @@ gfits_define_bintable_column (&theader, "E", "raErr", NULL, NULL, 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "decErr", NULL, NULL, 1.0, 0.0); - gfits_define_bintable_column (&theader, "E", "zp", NULL, NULL, 1.0, 0.0); - gfits_define_bintable_column (&theader, "E", "zpFactor", NULL, NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "zpPSF", NULL, NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "zpFactorPSF", NULL, NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "zpAPER", NULL, NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "zpFactorAPER", NULL, NULL, 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "telluricExt", NULL, NULL, 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "airmass", NULL, NULL, 1.0, 0.0); @@ -208,6 +216,8 @@ float *raErr ; ALLOCATE (raErr , float, Ndetections); float *decErr ; ALLOCATE (decErr , float, Ndetections); - float *zp ; ALLOCATE (zp , float, Ndetections); - float *zpFactor ; ALLOCATE (zpFactor , float, Ndetections); + float *zpPSF ; ALLOCATE (zpPSF , float, Ndetections); + float *zpFactorPSF ; ALLOCATE (zpFactorPSF , float, Ndetections); + float *zpAPER ; ALLOCATE (zpAPER , float, Ndetections); + float *zpFactorAPER; ALLOCATE (zpFactorAPER, float, Ndetections); float *telluricExt ; ALLOCATE (telluricExt , float, Ndetections); float *airmass ; ALLOCATE (airmass , float, Ndetections); @@ -234,6 +244,8 @@ raErr[i] = detections[i].raErr ; decErr[i] = detections[i].decErr ; - zp[i] = detections[i].zp ; - zpFactor[i] = detections[i].zpFactor ; + zpPSF[i] = detections[i].zpPSF ; + zpFactorPSF[i] = detections[i].zpFactorPSF ; + zpAPER[i] = detections[i].zpAPER ; + zpFactorAPER[i]= detections[i].zpFactorAPER; telluricExt[i] = detections[i].telluricExt ; airmass[i] = detections[i].airmass ; @@ -260,6 +272,8 @@ gfits_set_bintable_column (&theader, &ftable, "raErr", raErr , Ndetections); gfits_set_bintable_column (&theader, &ftable, "decErr", decErr , Ndetections); - gfits_set_bintable_column (&theader, &ftable, "zp", zp , Ndetections); - gfits_set_bintable_column (&theader, &ftable, "zpFactor", zpFactor , Ndetections); + gfits_set_bintable_column (&theader, &ftable, "zpPSF", zpPSF , Ndetections); + gfits_set_bintable_column (&theader, &ftable, "zpFactorPSF", zpFactorPSF , Ndetections); + gfits_set_bintable_column (&theader, &ftable, "zpAPER", zpAPER , Ndetections); + gfits_set_bintable_column (&theader, &ftable, "zpFactorAPER",zpFactorAPER, Ndetections); gfits_set_bintable_column (&theader, &ftable, "telluricExt", telluricExt , Ndetections); gfits_set_bintable_column (&theader, &ftable, "airmass", airmass , Ndetections); @@ -284,6 +298,8 @@ free (raErr ); free (decErr ); - free (zp ); - free (zpFactor ); + free (zpPSF ); + free (zpFactorPSF ); + free (zpAPER ); + free (zpFactorAPER); free (telluricExt ); free (airmass ); Index: /branches/czw_branch/20170908/Ohana/src/dvopsps/src/initialize_dvopsps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvopsps/src/initialize_dvopsps.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvopsps/src/initialize_dvopsps.c (revision 40477) @@ -123,4 +123,10 @@ if ((N = get_argument (argc, argv, "-insert-remote"))) { SAVE_REMOTE = FALSE; + remove_argument (N, &argc, argv); + } + + TEST_MODE = FALSE; + if ((N = get_argument (argc, argv, "-test-mode"))) { + TEST_MODE = TRUE; remove_argument (N, &argc, argv); } @@ -361,4 +367,10 @@ } + TEST_MODE = FALSE; + if ((N = get_argument (argc, argv, "-test-mode"))) { + TEST_MODE = TRUE; + remove_argument (N, &argc, argv); + } + HOST_ID = 0; if ((N = get_argument (argc, argv, "-hostID"))) { Index: /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps.c (revision 40477) @@ -136,6 +136,7 @@ snprintf (tmpline, 1024, "%s -dbname %s", command, DATABASE_NAME); strcpy (command, tmpline); - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } + if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } + if (TEST_MODE) { snprintf (tmpline, 1024, "%s -test-mode", command); strcpy (command, tmpline); } + if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } fprintf (stderr, "command: %s\n", command); Index: /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps_catalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps_catalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps_catalog.c (revision 40477) @@ -43,9 +43,11 @@ } - int hasPS1 = FALSE; - for (int j = 0; !hasPS1 && (j < Nsecfilt); j++) { - if (secfilt[i*Nsecfilt + j].flags & hasPS1_flag) hasPS1 = TRUE; - } - if (!hasPS1) continue; // skip non-PS1 data + if (!TEST_MODE) { + int hasPS1 = FALSE; + for (int j = 0; !hasPS1 && (j < Nsecfilt); j++) { + if (secfilt[i*Nsecfilt + j].flags & hasPS1_flag) hasPS1 = TRUE; + } + if (!hasPS1) continue; // skip non-PS1 data + } off_t m = average[i].lensobjOffset; Index: /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c (revision 40477) @@ -19,4 +19,5 @@ ZeroPoint = GetZeroPoint(); + int Nsecfilt = GetPhotcodeNsecfilt (); int missingID = 0; @@ -51,5 +52,10 @@ if (measure->photcode >= PHOTCODE_END) continue; - assign_detection_values (&detections[Ndetections], measure, average); + int equivCode = GetPhotcodeEquivCodebyCode (measure->photcode); + int Nsec = GetPhotcodeNsec (equivCode); + + SecFilt *secfilt = (Nsec >= 0) ? &catalog->secfilt[Nsecfilt*i + Nsec] : NULL; + + assign_detection_values (&detections[Ndetections], measure, average, secfilt); Ndetections ++; @@ -97,4 +103,5 @@ ZeroPoint = GetZeroPoint(); + int Nsecfilt = GetPhotcodeNsecfilt (); INITTIME; @@ -127,7 +134,12 @@ if (measure[Nmeas].photcode >= PHOTCODE_END) continue; + int equivCode = GetPhotcodeEquivCodebyCode (measure[Nmeas].photcode); + int Nsec = GetPhotcodeNsec (equivCode); + + SecFilt *secfilt = (Nsec > -1) ? &catalog->secfilt[Nsecfilt*i + Nsec] : NULL; + // XXX check return status Detections detection; - assign_detection_values (&detection, &measure[Nmeas], &average[i]); + assign_detection_values (&detection, &measure[Nmeas], &average[i], secfilt); if (!insert_detections_mysql_detvalue (&buffer, &detection)) { @@ -183,6 +195,6 @@ PrintIOBuffer (buffer, "INSERT INTO dvoDetectionFull (objID, detectID, ippObjID, ippDetectID, imageID, catID, "); - PrintIOBuffer (buffer, "ra, dec_, raErr, decErr, zp, zpFactor, telluricExt, airmass, expTime, "); - PrintIOBuffer (buffer, "Mpsf, dMpsf, Mkron, dMkron, Map, dMap, flags, objflags) VALUES \n"); + PrintIOBuffer (buffer, "ra, dec_, raErr, decErr, zpPSF, zpFactorPSF, zpAPER, zpFactorAPER, telluricExt, airmass, expTime, "); + PrintIOBuffer (buffer, "Mpsf, dMpsf, Mkron, dMkron, Map, dMap, flags, objflags, filtflags) VALUES \n"); return TRUE; @@ -254,6 +266,8 @@ PRINT_FLOAT(buffer, detection->raErr, "%.6f, "); PRINT_FLOAT(buffer, detection->decErr, "%.6f, "); - PRINT_FLOAT(buffer, detection->zp, "%.6f, "); - PRINT_FLOAT(buffer, detection->zpFactor, "%.6e, "); + PRINT_FLOAT(buffer, detection->zpPSF, "%.6f, "); + PRINT_FLOAT(buffer, detection->zpFactorPSF, "%.6e, "); + PRINT_FLOAT(buffer, detection->zpAPER, "%.6f, "); + PRINT_FLOAT(buffer, detection->zpFactorAPER,"%.6e, "); PRINT_FLOAT(buffer, detection->telluricExt, "%.6f, "); PRINT_FLOAT(buffer, detection->airmass, "%.6f, "); @@ -268,5 +282,6 @@ PrintIOBuffer (buffer, "%u, ", detection->flags); // measure.flags - PrintIOBuffer (buffer, "%u),\n", detection->objflags); // average.flags + PrintIOBuffer (buffer, "%u, ", detection->objflags); // average.flags + PrintIOBuffer (buffer, "%u),\n", detection->filtflags); // secfilt.flags return TRUE; } @@ -303,5 +318,7 @@ } -int assign_detection_values (Detections *detection, Measure *measure, Average *average) { +int assign_detection_values (Detections *detection, Measure *measure, Average *average, SecFilt *secfilt) { + + // myAbort ("check on measure->McalPSF vs McalAPER"); PhotCode *code = GetPhotcodebyCode(measure->photcode); @@ -311,7 +328,9 @@ float Mflat = isfinite(measure->Mflat) ? measure->Mflat : 0.0; float nominalZP = code->C * 0.001 + code->K * (measure->airmass - 1); - float zp = nominalZP - measure->Mcal - Mflat; - float zpFactor = pow(10.0, -0.4*zp + 3.56); - float telluricExt = - measure->Mcal; + float zpPSF = nominalZP - measure->McalPSF - Mflat; + float zpFactorPSF = pow(10.0, -0.4*zpPSF + 3.56); + float zpAPER = nominalZP - measure->McalAPER - Mflat; + float zpFactorAPER= pow(10.0, -0.4*zpAPER + 3.56); + float telluricExt = - measure->McalPSF; float expTime = pow(10.0, 0.4 * measure->dt); float airmass = measure->airmass; @@ -329,6 +348,8 @@ detection->decErr = measure->dYccd * 0.01 * fabs(measure->pltscale); // estimate of decErr - detection->zp = zp; - detection->zpFactor = zpFactor; + detection->zpPSF = zpPSF; + detection->zpFactorPSF = zpFactorPSF; + detection->zpAPER = zpAPER; + detection->zpFactorAPER = zpFactorAPER; detection->telluricExt = telluricExt; detection->airmass = airmass; @@ -337,7 +358,7 @@ // XXX clean this up with dvo_photcode_ops calls: // if (isfinite(measure->FluxPSF) && (measure->FluxPSF < 0.0)) - detection->Mpsf = getMagFromValueOrFlux (measure->FluxPSF, measure->M, zp); - detection->Mkron = getMagFromValueOrFlux (measure->FluxKron, measure->Mkron, zp); - detection->Map = getMagFromValueOrFlux (measure->FluxAp, measure->Map, zp); + detection->Mpsf = getMagFromValueOrFlux (measure->FluxPSF, measure->M, zpPSF); + detection->Mkron = getMagFromValueOrFlux (measure->FluxKron, measure->Mkron, zpAPER); + detection->Map = getMagFromValueOrFlux (measure->FluxAp, measure->Map, zpAPER); detection->dMpsf = getdMagFromValueOrFlux (measure->FluxPSF, measure->dFluxPSF, measure->dM); @@ -347,6 +368,7 @@ detection->flags = measure->dbFlags; // flags detection->objflags = average->flags; // flags - - return TRUE; -} - + detection->filtflags = secfilt ? secfilt->flags : 0; // flags + + return TRUE; +} + Index: /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_objects_dvopsps_catalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_objects_dvopsps_catalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/dvopsps/src/insert_objects_dvopsps_catalog.c (revision 40477) @@ -1,4 +1,6 @@ # include "dvopsps.h" # define DEBUG 0 + +int Format_PSX_Name (char *buffer, int Nbuffer, double tmp_ra, double tmp_dec); int insert_objects_dvopsps_catalog (Catalog *catalog, char *basename, MYSQL *mysql) { @@ -110,6 +112,6 @@ "RA_ERR FLOAT, " "DEC_ERR FLOAT, " - "IAU_NAME VARCHAR(32), " "PSO_NAME VARCHAR(32), " + "PSX_NAME VARCHAR(32), " "CHISQ_POS FLOAT, " "CHISQ_PM FLOAT, " @@ -219,6 +221,6 @@ "RA_ERR, " "DEC_ERR, " - "IAU_NAME, " "PSO_NAME, " + "PSX_NAME, " "CHISQ_POS, " "CHISQ_PM, " @@ -267,5 +269,7 @@ double Tmean = average->Tmean == 0 ? NAN : ohana_sec_to_mjd (average->Tmean); -// XXX this bit could/should be autocoded... + char coord_buffer[128]; + + // XXX this bit could/should be autocoded... PrintIOBuffer (ave_buffer, " ("); PrintIOBuffer (ave_buffer, OFF_T_FMT", ", average->extID); @@ -288,70 +292,15 @@ PRINT_FLOAT(ave_buffer, average->dD, "%.5f, "); // 0.010 mas precision - // Add names. - int ra_hr,ra_min,ra_sec,ra_fracsec; - int dec_deg,dec_min,dec_sec,dec_fracsec; - double tmp_ra,tmp_dec; - - if ((secfilt->NstackDet != 0)&&(isfinite(average->Rstk))&&(isfinite(average->Dstk))) { - tmp_ra = average->Rstk; - tmp_dec = average->Dstk; - } - else { - tmp_ra = average->R; - tmp_dec = average->D; - } - - double dec_sign = (tmp_dec == 0.0) ? 0.0 : tmp_dec / fabs(tmp_dec); - tmp_dec = fabs(tmp_dec); - - - tmp_ra /= 15.0; - ra_hr = (int) floor(tmp_ra); - - tmp_ra -= ra_hr; - tmp_ra *= 60.0; - ra_min = (int) floor(tmp_ra); - - tmp_ra -= ra_min; - tmp_ra *= 60.0; - ra_sec = (int) floor(tmp_ra); - - tmp_ra -= ra_sec; - tmp_ra *= 1000.0; - ra_fracsec = (int) floor(tmp_ra); - - dec_deg = (int) floor(tmp_dec); - - tmp_dec -= dec_deg; - tmp_dec *= 60.0; - dec_min = (int) floor(tmp_dec); - - tmp_dec -= dec_min; - tmp_dec *= 60.0; - dec_sec = (int) floor(tmp_dec); - - tmp_dec -= dec_sec; - tmp_dec *= 1000.0; - dec_fracsec = (int) floor(tmp_dec); - - dec_deg *= (int) dec_sign; - - // IAU NAME - PrintIOBuffer(ave_buffer, "'PSO J%02d%02d%02d.%03d%+03d%02d%02d.%03d', ", - ra_hr,ra_min,ra_sec,ra_fracsec, - dec_deg,dec_min,dec_sec,dec_fracsec); - - // PSO NAME - if ((secfilt->NstackDet != 0)&&(isfinite(average->Rstk))&&(isfinite(average->Dstk))) { - tmp_ra = average->Rstk; - tmp_dec = average->Dstk; - } - else { - tmp_ra = average->R; - tmp_dec = average->D; - } - - PrintIOBuffer(ave_buffer, "'PSO J%09.5f%+09.5f', ", - tmp_ra,tmp_dec); + int useStack = (!isfinite(average->R) || !isfinite(average->D)) && (secfilt->NstackDet != 0) && isfinite(average->Rstk) && isfinite(average->Dstk); + + double tmp_ra = useStack ? average->Rstk : average->R; + double tmp_dec = useStack ? average->Dstk : average->D; + + // PSO NAME (IAU approved) + PrintIOBuffer(ave_buffer, "'PSO J%08.4f%+08.4f', ", tmp_ra, tmp_dec); + + // PSX NAME + Format_PSX_Name (coord_buffer, 128, tmp_ra, tmp_dec); + PrintIOBuffer(ave_buffer, "'%s', ", coord_buffer); // End names @@ -372,8 +321,8 @@ float meanPSFMagMin = NAN; float meanPSFMagMax = NAN; - if (isfinite(secfilt->dM) && isfinite(secfilt->M)) { - meanPSFMag = secfilt->M; - meanPSFMagErr = secfilt->dM; - meanPSFMagStd = secfilt->Mstdev; + if (isfinite(secfilt->dMpsfChp) && isfinite(secfilt->MpsfChp)) { + meanPSFMag = secfilt->MpsfChp; + meanPSFMagErr = secfilt->dMpsfChp; + meanPSFMagStd = secfilt->sMpsfChp; meanPSFMagMin = secfilt->Mmin; meanPSFMagMax = secfilt->Mmax; @@ -383,8 +332,8 @@ float meanKronMagErr = NAN; float meanKronMagStd = NAN; - if (isfinite(secfilt->dMkron) && isfinite(secfilt->Mkron) && (secfilt->dMkron < 0.333)) { - meanKronMag = secfilt->Mkron; - meanKronMagErr = secfilt->dMkron; - meanKronMagStd = secfilt->sMkron; + if (isfinite(secfilt->dMkronChp) && isfinite(secfilt->MkronChp) && (secfilt->dMkronChp < 0.333)) { + meanKronMag = secfilt->MkronChp; + meanKronMagErr = secfilt->dMkronChp; + meanKronMagStd = secfilt->sMkronChp; } @@ -392,8 +341,8 @@ float meanApMagErr = NAN; float meanApMagStd = NAN; - if (isfinite(secfilt->dMap) && isfinite(secfilt->Map)) { - meanApMag = secfilt->Map; - meanApMagErr = secfilt->dMap; - meanApMagStd = secfilt->sMap; + if (isfinite(secfilt->dMapChp) && isfinite(secfilt->MapChp)) { + meanApMag = secfilt->MapChp; + meanApMagErr = secfilt->dMapChp; + meanApMagStd = secfilt->sMapChp; } @@ -524,2 +473,40 @@ return TRUE; } + +// XXX if the coords have NAN values (not sure why that would happen), return FALSE? +int Format_PSX_Name (char *buffer, int Nbuffer, double tmp_ra, double tmp_dec) { + + // Add names. + int ra_hr, ra_min, dec_deg, dec_min; + float ra_sec, dec_sec; + + // convert to hours: + tmp_ra /= 15.0; + ra_hr = (int) floor(tmp_ra); + + tmp_ra -= ra_hr; + tmp_ra *= 60.0; + ra_min = (int) floor(tmp_ra); + + tmp_ra -= ra_min; + tmp_ra *= 60.0; + ra_sec = trunc(100.0*tmp_ra) / 100.0; // ensure truncation for XX.XX + + char dec_sign = (tmp_dec >= 0.0) ? '+' : '-'; + tmp_dec = fabs(tmp_dec); + + dec_deg = (int) floor(tmp_dec); + + tmp_dec -= dec_deg; + tmp_dec *= 60.0; + dec_min = (int) floor(tmp_dec); + + tmp_dec -= dec_min; + tmp_dec *= 60.0; + dec_sec = trunc(10.0*tmp_dec) / 10.0; // ensure truncation for XX.X + + // PSX NAME + snprintf (buffer, Nbuffer, "PSX J%02d%02d%05.2f%c%02d%02d%04.1f", ra_hr, ra_min, ra_sec, dec_sign, dec_deg, dec_min, dec_sec); + + return TRUE; +} Index: /branches/czw_branch/20170908/Ohana/src/fakeastro/src/insert_fakestar.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/fakeastro/src/insert_fakestar.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/fakeastro/src/insert_fakestar.c (revision 40477) @@ -93,9 +93,9 @@ // I need a photcode for r-band - catalog[0].secfilt[Nave*Nsecfilt+0].M = m_g; - catalog[0].secfilt[Nave*Nsecfilt+1].M = m_r; - catalog[0].secfilt[Nave*Nsecfilt+2].M = m_i; - catalog[0].secfilt[Nave*Nsecfilt+3].M = m_z; - catalog[0].secfilt[Nave*Nsecfilt+4].M = m_y; + catalog[0].secfilt[Nave*Nsecfilt+0].MpsfChp = m_g; + catalog[0].secfilt[Nave*Nsecfilt+1].MpsfChp = m_r; + catalog[0].secfilt[Nave*Nsecfilt+2].MpsfChp = m_i; + catalog[0].secfilt[Nave*Nsecfilt+3].MpsfChp = m_z; + catalog[0].secfilt[Nave*Nsecfilt+4].MpsfChp = m_y; catalog[0].starpar[Nstarpar] = stars[i].starpar; Index: /branches/czw_branch/20170908/Ohana/src/fakeastro/src/load_template_images.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/fakeastro/src/load_template_images.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/fakeastro/src/load_template_images.c (revision 40477) @@ -61,5 +61,6 @@ refimage[Nrefimage].secz = childImage->secz; - refimage[Nrefimage].Mcal = childImage->Mcal; + refimage[Nrefimage].McalPSF = childImage->McalPSF; + refimage[Nrefimage].McalAPER = childImage->McalAPER; refimage[Nrefimage].dMcal = childImage->dMcal; refimage[Nrefimage].exptime = childImage->exptime; Index: /branches/czw_branch/20170908/Ohana/src/fakeastro/src/make_2mass_measures.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/fakeastro/src/make_2mass_measures.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/fakeastro/src/make_2mass_measures.c (revision 40477) @@ -44,16 +44,16 @@ // make a crude JHK color for now: - double J_PS1 = secfilt[i*Nsecfilt + Nsec].M - 1.0; // make all stars have y-J = 1.0 + double J_PS1 = secfilt[i*Nsecfilt + Nsec].MpsfChp - 1.0; // make all stars have y-J = 1.0 if (J_PS1 > MAX_MAG_2MASS) continue; // only generate 2MASS detections for objects with J_PS1 < 16.0 - if (isnan(secfilt[i*Nsecfilt + Nsec].M)) { + if (isnan(secfilt[i*Nsecfilt + Nsec].MpsfChp)) { // look for a non-NAN secfilt mag and just use that (it is not super important) int ns; for (ns = 0; ns < Nsecfilt; ns++) { - if (!isnan(secfilt[i*Nsecfilt + ns].M)) break; + if (!isnan(secfilt[i*Nsecfilt + ns].MpsfChp)) break; } if (ns == Nsecfilt) continue; // no non-nan - J_PS1 = secfilt[i*Nsecfilt + ns].M; // pretend secfilt.M = J + J_PS1 = secfilt[i*Nsecfilt + ns].MpsfChp; // pretend secfilt.MpsfChp = J if (J_PS1 > MAX_MAG_2MASS) continue; // only generate 2MASS detections for objects with J_PS1 < 16.0 } @@ -103,5 +103,6 @@ measure[Nmeasure].airmass = 1.0; measure[Nmeasure].az = 0.0; // irrelevant - measure[Nmeasure].Mcal = 0.0; + measure[Nmeasure].McalPSF = 0.0; + measure[Nmeasure].McalAPER= 0.0; measure[Nmeasure].t = tzero_2mass; measure[Nmeasure].dt = 0.0; Index: /branches/czw_branch/20170908/Ohana/src/fakeastro/src/make_fake_stars_catalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/fakeastro/src/make_fake_stars_catalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/fakeastro/src/make_fake_stars_catalog.c (revision 40477) @@ -42,6 +42,6 @@ // XXX put in airmass? - float ZP = SCALE*code->C - image->Mcal + Mtime; - float ZPo = 25.0 - image->Mcal + Mtime; + float ZP = SCALE*code->C - image->McalPSF + Mtime; + float ZPo = 25.0 - image->McalPSF + Mtime; // float ZP = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - measure[0].Mcal; @@ -79,5 +79,5 @@ // float Minst = Msky - SCALE*code->C - measure[0].dt - double Minst = secfilt[i*Nsecfilt + Nsec].M - ZP; + double Minst = secfilt[i*Nsecfilt + Nsec].MpsfChp - ZP; double Counts = pow(10.0, -0.4*Minst); double SkyCts = sky[Nsec]; @@ -175,5 +175,6 @@ stars[Nstars].measure.az = azimuth (15.0*image->sidtime - stars[Nstars].average.R, stars[Nstars].average.D, image->latitude); - stars[Nstars].measure.Mcal = image->Mcal; + stars[Nstars].measure.McalPSF = image->McalPSF; + stars[Nstars].measure.McalAPER= image->McalAPER; stars[Nstars].measure.t = image->tzero + 1e-4*stars[Nstars].measure.Yccd*image->trate; // trate is in 0.1 msec / row stars[Nstars].measure.dt = Mtime; Index: /branches/czw_branch/20170908/Ohana/src/fakeastro/src/make_gaia_measures.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/fakeastro/src/make_gaia_measures.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/fakeastro/src/make_gaia_measures.c (revision 40477) @@ -42,16 +42,16 @@ // make a crude G-i color for now: - double G_PS1 = secfilt[i*Nsecfilt + Nsec].M; // make all stars have G-i = 1.0 + double G_PS1 = secfilt[i*Nsecfilt + Nsec].MpsfChp; // make all stars have G-i = 1.0 if (G_PS1 > MAX_MAG_GAIA) continue; // only generate GAIA detections for objects with G_PS1 < 21.0 - if (isnan(secfilt[i*Nsecfilt + Nsec].M)) { + if (isnan(secfilt[i*Nsecfilt + Nsec].MpsfChp)) { // look for a non-NAN secfilt mag and just use that (it is not super important) int ns; for (ns = 0; ns < Nsecfilt; ns++) { - if (!isnan(secfilt[i*Nsecfilt + ns].M)) break; + if (!isnan(secfilt[i*Nsecfilt + ns].MpsfChp)) break; } if (ns == Nsecfilt) continue; // no non-nan - G_PS1 = secfilt[i*Nsecfilt + ns].M; // pretend secfilt.M = G + G_PS1 = secfilt[i*Nsecfilt + ns].MpsfChp; // pretend secfilt.M = G if (G_PS1 > MAX_MAG_GAIA) continue; // only generate GAIA detections for objects with G_PS1 < 21.0 } @@ -104,5 +104,6 @@ measure[Nmeasure].airmass = 1.0; measure[Nmeasure].az = 0.0; // irrelevant - measure[Nmeasure].Mcal = 0.0; + measure[Nmeasure].McalPSF = 0.0; + measure[Nmeasure].McalAPER= 0.0; measure[Nmeasure].t = tzero_gaia; measure[Nmeasure].dt = 0.0; Index: /branches/czw_branch/20170908/Ohana/src/getstar/src/ReadImageHeader.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/getstar/src/ReadImageHeader.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/getstar/src/ReadImageHeader.c (revision 40477) @@ -127,6 +127,7 @@ /* secz is in units milli-airmass */ - image[0].Mcal = 0.0; - image[0].Xm = NAN_S_SHORT; + image[0].McalPSF = 0.0; + image[0].McalAPER = 0.0; + image[0].McalChiSq = NAN; image[0].flags = 0; Index: /branches/czw_branch/20170908/Ohana/src/getstar/src/select_by_region.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/getstar/src/select_by_region.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/getstar/src/select_by_region.c (revision 40477) @@ -78,5 +78,5 @@ } } else { - mag = catalog[0].secfilt[i*Nsecfilt + Nsec].M; + mag = catalog[0].secfilt[i*Nsecfilt + Nsec].MpsfChp; } if (isnan(mag)) continue; @@ -131,5 +131,5 @@ } } else { - mag = catalog[0].secfilt[i*Nsecfilt + Nsec].M; + mag = catalog[0].secfilt[i*Nsecfilt + Nsec].MpsfChp; } if (isnan(mag) || (mag > MagLimitValue)) continue; Index: /branches/czw_branch/20170908/Ohana/src/getstar/src/write_getstar_ps1_dev_0.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/getstar/src/write_getstar_ps1_dev_0.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/getstar/src/write_getstar_ps1_dev_0.c (revision 40477) @@ -55,5 +55,5 @@ // find primary magnitude if (Nsec_c0 != -1) { - output[i].mag = secfilt[i*Nsecfilt + Nsec_c0].M; + output[i].mag = secfilt[i*Nsecfilt + Nsec_c0].MpsfChp; } else { output[i].mag = NAN; @@ -69,5 +69,5 @@ // find color term 1 if (Nsec_c1 != -1) { - output[i].c1 = secfilt[i*Nsecfilt + Nsec_c1].M; + output[i].c1 = secfilt[i*Nsecfilt + Nsec_c1].MpsfChp; } else { output[i].c1 = NAN; @@ -83,5 +83,5 @@ // find color term 2 if (Nsec_c2 != -1) { - output[i].c2 = secfilt[i*Nsecfilt + Nsec_c2].M; + output[i].c2 = secfilt[i*Nsecfilt + Nsec_c2].MpsfChp; } else { output[i].c2 = NAN; Index: /branches/czw_branch/20170908/Ohana/src/getstar/src/write_getstar_ps1_dev_1.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/getstar/src/write_getstar_ps1_dev_1.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/getstar/src/write_getstar_ps1_dev_1.c (revision 40477) @@ -58,5 +58,5 @@ // find primary magnitude if (Nsec_c0 != -1) { - output[i].mag = secfilt[i*Nsecfilt + Nsec_c0].M; + output[i].mag = secfilt[i*Nsecfilt + Nsec_c0].MpsfChp; } else { output[i].mag = NAN; @@ -72,5 +72,5 @@ // find color term 1 if (Nsec_c1 != -1) { - output[i].c1 = secfilt[i*Nsecfilt + Nsec_c1].M; + output[i].c1 = secfilt[i*Nsecfilt + Nsec_c1].MpsfChp; } else { output[i].c1 = NAN; @@ -86,5 +86,5 @@ // find color term 2 if (Nsec_c2 != -1) { - output[i].c2 = secfilt[i*Nsecfilt + Nsec_c2].M; + output[i].c2 = secfilt[i*Nsecfilt + Nsec_c2].MpsfChp; } else { output[i].c2 = NAN; Index: /branches/czw_branch/20170908/Ohana/src/getstar/src/write_getstar_ps1_dev_2.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/getstar/src/write_getstar_ps1_dev_2.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/getstar/src/write_getstar_ps1_dev_2.c (revision 40477) @@ -64,5 +64,5 @@ // find primary magnitude if (Nsec_c0 != -1) { - output[i].mag = secfilt[i*Nsecfilt + Nsec_c0].M; + output[i].mag = secfilt[i*Nsecfilt + Nsec_c0].MpsfChp; } else { output[i].mag = NAN; @@ -78,5 +78,5 @@ // find color term 1 if (Nsec_c1 != -1) { - output[i].c1 = secfilt[i*Nsecfilt + Nsec_c1].M; + output[i].c1 = secfilt[i*Nsecfilt + Nsec_c1].MpsfChp; } else { output[i].c1 = NAN; @@ -92,5 +92,5 @@ // find color term 2 if (Nsec_c2 != -1) { - output[i].c2 = secfilt[i*Nsecfilt + Nsec_c2].M; + output[i].c2 = secfilt[i*Nsecfilt + Nsec_c2].MpsfChp; } else { output[i].c2 = NAN; Index: /branches/czw_branch/20170908/Ohana/src/imregister/imphot/dumpfits.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/imregister/imphot/dumpfits.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/imregister/imphot/dumpfits.c (revision 40477) @@ -114,5 +114,5 @@ startstr = ohana_sec_to_date (subset[0].tzero); filtstr = GetPhotcodeNamebyCode (subset[0].photcode); - zp = subset[0].Mcal; + zp = subset[0].McalPSF; dzp = subset[0].dMcal; XY_to_RD (&ra, &dec, 0.0, 0.0, &subset[0].coords); Index: /branches/czw_branch/20170908/Ohana/src/imregister/imphot/output.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/imregister/imphot/output.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/imregister/imphot/output.c (revision 40477) @@ -37,5 +37,5 @@ fprintf (stdout, "%s %s %s %7.4f %7.4f %7.4f %5d %02x\n", image[i].name, photstr, timestr, - image[i].Mcal, image[i].dMcal, image[i].secz, image[i].nstar, image[i].flags); + image[i].McalPSF, image[i].dMcal, image[i].secz, image[i].nstar, image[i].flags); free (timestr); } Index: /branches/czw_branch/20170908/Ohana/src/kapa2/include/constants.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/include/constants.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/include/constants.h (revision 40477) @@ -27,5 +27,5 @@ # define MIN_WIDTH 50 # define MIN_HEIGHT 50 -# define LABEL_MAXLEN 128 +# define LABEL_MAXLEN 512 typedef enum { Index: /branches/czw_branch/20170908/Ohana/src/kapa2/include/prototypes.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/include/prototypes.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/include/prototypes.h (revision 40477) @@ -36,4 +36,5 @@ int DrawObjectN PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); void DrawPoints PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); +void DrawBars PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *object, int mode)); void ClipLine PROTO((Graphic *graphic, double x0, double y0, double x1, double y1, double X0, double Y0, double X1, double Y1)); void DrawXErrors PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); @@ -139,4 +140,5 @@ void bDrawConnect PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); void bDrawHistogram PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); +void bDrawBars PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object, int mode)); void bDrawPoints PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); void bDrawXErrors PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); Index: /branches/czw_branch/20170908/Ohana/src/kapa2/include/structures.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/include/structures.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/include/structures.h (revision 40477) @@ -115,7 +115,9 @@ typedef struct { int x, y; - int dx, dy; + int dx, dy; // unused? double angle; int size; + int justify; + int color; char font[64]; char text[LABEL_MAXLEN]; Index: /branches/czw_branch/20170908/Ohana/src/kapa2/src/DrawObjects.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/src/DrawObjects.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/src/DrawObjects.c (revision 40477) @@ -1,10 +1,11 @@ # include "Ximage.h" -void DrawBars (Graphic *graphic, KapaGraphWidget *graph, Gobjects *object, int mode); +/* DrawRectangle & FillRectangle : take rectangle lower-left corner and widths + DrawCircle & FillCircle : take circle center and radius +*/ # define DrawLine(X1,Y1,X2,Y2) (XDrawLine (graphic->display, graphic->window, graphic->gc, (int)(X1), (int)(Y1), (int)(X2), (int)(Y2))) -# define DrawRectangle(X1,Y1,X2,Y2) (XDrawRectangle (graphic->display, graphic->window, graphic->gc, (int)(X1), (int)(Y1), (int)(X2), (int)(Y2))) -# define FillRectangle(X1,Y1,X2,Y2) (XFillRectangle (graphic->display, graphic->window, graphic->gc, (int)(X1), (int)(Y1), (int)(X2), (int)(Y2))) -// # define FillRectangle(X1,Y1,X2,Y2) ; +# define DrawRectangle(X,Y,dX,dY) (XDrawRectangle (graphic->display, graphic->window, graphic->gc, (int)(X), (int)(Y), (int)(dX), (int)(dY))) +# define FillRectangle(X,Y,dX,dY) (XFillRectangle (graphic->display, graphic->window, graphic->gc, (int)(X), (int)(Y), (int)(dX), (int)(dY))) # define DrawCircle(X,Y,R) (XDrawArc (graphic->display, graphic->window, graphic->gc, (int)(X-R), (int)(Y-R), abs(2*R), abs(2*R), 0, 23040)) # define FillCircle(X,Y,R) (XFillArc (graphic->display, graphic->window, graphic->gc, (int)(X-R), (int)(Y-R), abs(2*R), abs(2*R), 0, 23040)) @@ -96,20 +97,26 @@ switch (object[0].style) { case KAPA_PLOT_CONNECT: + // fprintf (stderr, "plot KAPA_PLOT_CONNECT: \n"); DrawConnect (graphic, graph, object); break; case KAPA_PLOT_HISTOGRAM: + // fprintf (stderr, "plot KAPA_PLOT_HISTOGRAM:\n"); DrawHistogram (graphic, graph, object); break; case KAPA_PLOT_BARS_SOLID: + // fprintf (stderr, "plot KAPA_PLOT_BARS_SOLID:\n"); DrawBars (graphic, graph, object, KAPA_PLOT_BARS_SOLID); break; case KAPA_PLOT_BARS_OUTLINE: + // fprintf (stderr, "plot KAPA_PLOT_BARS_OUTLINE:\n"); DrawBars (graphic, graph, object, KAPA_PLOT_BARS_OUTLINE); break; case KAPA_PLOT_BARS_OUTFILL: + // fprintf (stderr, "plot KAPA_PLOT_BARS_OUTFILL:\n"); DrawBars (graphic, graph, object, KAPA_PLOT_BARS_OUTFILL); break; case KAPA_PLOT_POINTS: default: + // fprintf (stderr, "plot KAPA_PLOT_POINTS:\n"); DrawPoints (graphic, graph, object); break; @@ -162,7 +169,5 @@ sx1 = x[i]*mxi + y[i]*mxj + bx + XCENTER; sy1 = x[i]*myi + y[i]*myj + by + YCENTER; - ClipLine (graphic, sx0, sy0, sx1, sy1, X0, Y0, X1, Y1); - /* DrawLine (sx0, sy0, sx1, sy1); */ sx0 = sx1; sy0 = sy1; } @@ -225,5 +230,4 @@ /******/ /* simplify the code abit by finding triplets, watch out for a histogram of 2 points */ -# if (1) void DrawHistogram (Graphic *graphic, KapaGraphWidget *graph, Gobjects *object) { @@ -309,114 +313,4 @@ DrawLine (sxa, sy1, sxa, sya); } - -# else - -void DrawHistogram (Graphic *graphic, KapaGraphWidget *graph, Gobjects *object) { - - int i; - float *x, *y; - double mxi, mxj, myi, myj, bxi, bxj, byi, byj, bx, by; - double sx0, sy0, sx1, sy1, sxa, sya, sxo, syo; - double X0, X1, Y0, Y1; - - mxi = graph[0].axis[0].dfx / (object[0].x1 - object[0].x0); - mxj = graph[0].axis[1].dfx / (object[0].y1 - object[0].y0); - myi = graph[0].axis[0].dfy / (object[0].x1 - object[0].x0); - myj = graph[0].axis[1].dfy / (object[0].y1 - object[0].y0); - - bxi = graph[0].axis[0].fx - object[0].x0*graph[0].axis[0].dfx/(object[0].x1 - object[0].x0); - bxj = -object[0].y0*graph[0].axis[1].dfx/(object[0].y1 - object[0].y0); - byi = -object[0].x0*graph[0].axis[0].dfy/(object[0].x1 - object[0].x0); - byj = graph[0].axis[1].fy - object[0].y0*graph[0].axis[1].dfy/(object[0].y1 - object[0].y0); - - bx = bxi + bxj; - by = byi + byj; - - X0 = graph[0].axis[0].fx; - X1 = graph[0].axis[0].fx + graph[0].axis[0].dfx; - Y0 = graph[0].axis[1].fy; - Y1 = graph[0].axis[1].fy + graph[0].axis[1].dfy; - - /* find the first valid datapoint */ - x = object[0].x; y = object[0].y; - - /* we only worry about pairs of points of which one is on the screen. for each such - pair, we draw the connecting lines from sx0,sy0 to sx1,sy1, passing through the point - halfway between. - - if, for a given point, there was no valid x0,y0, then we call this the first point - save both the current and previous points. - */ - - int firstPair = TRUE; - sx0r = sy0r = NAN; - sxPr = syPr = NAN; - for (i = 0; i < object[0].Npts; i++) { - if (!(finite(x[i]) && finite(y[i]))) continue; - sx1r = x[i]*mxi + y[i]*mxj + bx + XCENTER; - sy1r = x[i]*myi + y[i]*myj + by + YCENTER; - - // we have hit the first valid point, but not yet the second one - if (!isfinite (sx0r)) { - sx0r = sx1r; sy0r = sy1r; - continue; - } - - if ((sx1r < X0) && (sx0r < X0)) { - // both points to left, skip the pair - sxPr = sx0r; syPr = sy0r; - sx0r = sx1r; sy0r = sy1r; - continue; - } - if ((sx1r > X0) && (sx0r > X0)) { - // both points to right, skip the pair - sxPr = sx0r; syPr = sy0r; - sx0r = sx1r; sy0r = sy1r; - continue; - } - - // saturated values for sx0,sy0 - sx0 = MIN (MAX (sx0r, X0), X1); - sy0 = MAX (MIN (sy0r, Y0), Y1); - - // saturated values for sx1,sy1 - sx1 = MIN (MAX (sx1r, X0), X1); - sy1 = MAX (MIN (sy1r, Y0), Y1); - - if (firstPair) { - /* draw segment equal distance behind first point and down to x-axis */ - sxar = sx0r - 0.5*(sx1r - sx0r); - sxa = MIN (MAX (sxar, X0), X1); - sya = MAX (sy0r, Y0); - XSetForeground (graphic->display, graphic->gc, graphic->fore); - DrawLine (sx0, sy0, sxa, sy0); - DrawLine (sxa, sy0, sxa, sya); - XSetForeground (graphic->display, graphic->gc, graphic->color[object[0].color]); - FillRectangle (sxa + 3, sy0 + 3, sx0 - sxa - 6, sya - sy0 - 6); - firstPair = FALSE; - } - - // midpoint & saturated version - sxar = 0.5*(sx0r + sx1r); - sxa = MIN (MAX (sxar, X0), X1); - - XSetForeground (graphic->display, graphic->gc, graphic->fore); - DrawLine (sx0, sy0, sxa, sy0); - DrawLine (sxa, sy0, sxa, sy1); - DrawLine (sxa, sy1, sx1, sy1); - XSetForeground (graphic->display, graphic->gc, graphic->color[object[0].color]); - FillRectangle (sx0 + 3, sy0 + 3, sxa - sx0 - 6, Y0 - sy0 - 6); - FillRectangle (sxa + 3, sy1 + 3, sx1 - sxa - 6, Y0 - sy1 - 6); - sxPr = sx0r; syPr = sy0r; - sx0r = sx1r; sy0r = sy1r; - } - - /* draw segment equal distance after last point and down to x-axis */ - sxa = MIN (MAX (sx1 + 0.5*(sx1 - sxo), X0), X1); - sya = MAX (sy1, Y0); - DrawLine (sx1, sy1, sxa, sy1); - DrawLine (sxa, sy1, sxa, sya); -} -# endif // uses object->color @@ -498,14 +392,14 @@ /* we are drawing bars which are filled rectangles of height y[i] and width 0.5*dx one point at a time + + I need to know the distance to the next and prev points to calculate dx + for the first point, dx is x[1] - x[0] + for the last point, dx is x[-1] - x[-2] (x[-1] is the last point) + for the rest, dx is 0.5*(x[i+1] - x[i-1]) + + rather than working out complex on-the-fly logic, I want to find the first and last + valid point in an initial pass, then calculate the above for the remainder. + TBD: make an index vector of only valid points? */ - - // I need to know the distance to the next and prev points to calculate dx - // for the first point, dx is x[1] - x[0] - // for the last point, dx is x[-1] - x[-2] (x[-1] is the last point) - // for the rest, dx is 0.5*(x[i+1] - x[i-1]) - - // rather than working out complex on-the-fly logic, I want to find the first and last - // valid point in an initial pass, the calculate the above for the remainder. - // or make an index vector of only valid points? int Ngood = 0; @@ -522,5 +416,4 @@ // coordinate on the screen of the point: double sx1r = x[i]*mxi + y[i]*mxj + bx + XCENTER; - // double sy1r = x[i]*myi + y[i]*myj + by + YCENTER; if (sx1r < X0) { @@ -576,5 +469,4 @@ n = goodPoint[1]; double sx1r = x[n]*mxi + y[n]*mxj + bx + XCENTER; -// double sy1r = x[n]*myi + y[n]*myj + by + YCENTER; n = goodPoint[0]; @@ -605,11 +497,10 @@ n = goodPoint[i + 1]; double sx2r = x[n]*mxi + y[n]*mxj + bx + XCENTER; - // double sy2r = x[n]*myi + y[n]*myj + by + YCENTER; n = goodPoint[i - 1]; double sx0r = x[n]*mxi + y[n]*mxj + bx + XCENTER; - // double sy0r = x[n]*myi + y[n]*myj + by + YCENTER; - - double dx = 0.5*0.5*object->size*(sx2r - sx0r); // 2 factors of 0.5 (we want half of the average spacing) + + // below we have 2 factors of 0.5 (we want half of the average spacing) + double dx = 0.5*0.5*object->size*(sx2r - sx0r); n = goodPoint[i]; @@ -642,5 +533,4 @@ n = goodPoint[Ngood - 2]; double sx0r = x[n]*mxi + y[n]*mxj + bx + XCENTER; -// double sy0r = x[n]*myi + y[n]*myj + by + YCENTER; double dx = 0.5*object->size*(sx1r - sx0r); Index: /branches/czw_branch/20170908/Ohana/src/kapa2/src/DrawTextlines.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/src/DrawTextlines.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/src/DrawTextlines.c (revision 40477) @@ -12,13 +12,13 @@ fontname = GetRotFont (&size); XSetForeground (graphic->display, graphic->gc, graphic->fore); - DrawRotTextInit (graphic->display, graphic->window, graphic->gc, graphic->fore, graphic->back); for (i = 0; i < graph[0].Ntextline; i++) { if (strcmp (graph[0].textline[i].text, "")) { + DrawRotTextInit (graphic->display, graphic->window, graphic->gc, graphic->color[graph[0].textline[i].color], graphic->back); angle = graph[0].textline[i].angle; x = graph[0].textline[i].x; y = graph[0].textline[i].y; SetRotFont (graph[0].textline[i].font, graph[0].textline[i].size); - DrawRotText (x, y, graph[0].textline[i].text, 5, angle); + DrawRotText (x, y, graph[0].textline[i].text, graph[0].textline[i].justify, angle); } } Index: /branches/czw_branch/20170908/Ohana/src/kapa2/src/LoadObject.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/src/LoadObject.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/src/LoadObject.c (revision 40477) @@ -42,5 +42,5 @@ graph[0].objects[N].color = KapaColormapSize() - 1; } - if ((graph[0].objects[N].style != 2) && (graph[0].objects[N].color < 0)) { + if ((graph[0].objects[N].style != KAPA_PLOT_POINTS) && (graph[0].objects[N].color < 0)) { graph[0].objects[N].color = 0; } Index: /branches/czw_branch/20170908/Ohana/src/kapa2/src/LoadTextlines.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/src/LoadTextlines.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/src/LoadTextlines.c (revision 40477) @@ -4,5 +4,5 @@ char *string; - int N, size; + int N, size, justify, color; double tX, tY, tT, L; Section *section; @@ -23,5 +23,5 @@ REALLOCATE (graph[0].textline, Label, graph[0].Ntextline); - KiiScanMessage (sock, "%lf %lf %lf", &tX, &tY, &tT); + KiiScanMessage (sock, "%lf %lf %lf %d %d", &tX, &tY, &tT, &justify, &color); L = graph[0].axis[0].dfx; @@ -52,4 +52,6 @@ graph[0].textline[N].size = size; strcpy (graph[0].textline[N].font, string); + graph[0].textline[N].justify = justify; + graph[0].textline[N].color = color; if (USE_XWINDOW) DrawTextlines (graph); Index: /branches/czw_branch/20170908/Ohana/src/kapa2/src/PSObjects.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/src/PSObjects.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/src/PSObjects.c (revision 40477) @@ -1,7 +1,13 @@ # include "Ximage.h" +void PSBars (KapaGraphWidget *graph, Gobjects *object, FILE *f, int mode); + +/* DrawRectangle & FillRectangle : take rectangle lower-left corner and widths + DrawCircle & FillCircle : take circle center and radius +*/ + # define DrawLine(X1,Y1,X2,Y2) (fprintf (f, " %6.2f %6.2f %6.2f %6.2f L\n", X1, graphic->dy - Y1, X2, graphic->dy - Y2)) -# define DrawRectangle(X,Y,dX,dY) (fprintf (f, " %6.2f %6.2f %6.2f %6.2f B\n", (dX), (dY), (X-0.5*dX), (graphic->dy-Y-0.5*dY))) -# define FillRectangle(X,Y,dX,dY) (fprintf (f, " %6.2f %6.2f %6.2f %6.2f F\n", (dX), (dY), (X-0.5*dX), (graphic->dy-Y-0.5*dY))) +# define DrawRectangle(X,Y,dX,dY) (fprintf (f, " %6.2f %6.2f %6.2f %6.2f B\n", (dX), (dY), (X), (graphic->dy-Y))) +# define FillRectangle(X,Y,dX,dY) (fprintf (f, " %6.2f %6.2f %6.2f %6.2f F\n", (dX), (dY), (X), (graphic->dy-Y))) # define DrawCircle(X1,Y1,R) (fprintf (f, " %6.2f %6.2f %6.2f C\n", (X1), (graphic->dy - Y1), (R))) # define FillCircle(X1,Y1,R) (fprintf (f, " %6.2f %6.2f %6.2f FC\n", (X1), (graphic->dy - Y1), (R))) @@ -32,6 +38,6 @@ } -int PSObjectsN (KapaGraphWidget *graph, Gobjects *object, FILE *f) { - +void PSLineStyle (KapaGraphWidget *graph, Gobjects *object, FILE *f) { + static char short_dash[] = "4 4"; static char long_dash[] = "8 8"; @@ -64,4 +70,9 @@ fprintf (f, "%s setrgbcolor\n", KapaColorRGBString(object->color)); } +} + +int PSObjectsN (KapaGraphWidget *graph, Gobjects *object, FILE *f) { + + PSLineStyle (graph, object, f); switch (object->style) { @@ -71,4 +82,13 @@ case KAPA_PLOT_HISTOGRAM: PSHistogram (graph, object, f); + break; + case KAPA_PLOT_BARS_SOLID: + PSBars (graph, object, f, KAPA_PLOT_BARS_SOLID); + break; + case KAPA_PLOT_BARS_OUTLINE: + PSBars (graph, object, f, KAPA_PLOT_BARS_OUTLINE); + break; + case KAPA_PLOT_BARS_OUTFILL: + PSBars (graph, object, f, KAPA_PLOT_BARS_OUTFILL); break; case KAPA_PLOT_POINTS: @@ -265,25 +285,246 @@ DrawLine (sx1, sy1, sxa, sy1); DrawLine (sxa, sy1, sxa, sya); - -# if (0) - sx0 = x[i]*mxi + y[i]*mxj + bx; - sy0 = x[i]*myi + y[i]*myj + by; - sx0 = MIN (MAX (sx0, X0), X1); - sy0 = MAX (MIN (sy0, Y0), Y1); - - /* continue with rest of points */ - for (i++; i < object[0].Npts; i++) { +} + +// uses object->color +# define HISTOGRAM_SOLID(X_VALUE, Y_VALUE, DX_VAL) { \ + /* histogram bar corners */ \ + double sxmin = (X_VALUE) - 0.5*(DX_VAL); \ + double sxmax = (X_VALUE) + 0.5*(DX_VAL); \ + double symin = Xaxis; \ + double symax = (Y_VALUE); \ + /* saturated values for corner coords: */ \ + sxmin = MIN (MAX (sxmin, X0), X1); \ + sxmax = MIN (MAX (sxmax, X0), X1); \ + symin = MAX (MIN (symin, Y0), Y1); \ + symax = MAX (MIN (symax, Y0), Y1); \ + double dy = fabs(symax - symin); \ + double ylow = MAX(symin, symax); \ + FillRectangle (sxmin, ylow, (DX_VAL), dy); } + +// uses object->color +# define HISTOGRAM_OUTLINE(X_VALUE, Y_VALUE, DX_VAL) { \ + /* histogram bar corners */ \ + double sxmin = (X_VALUE) - 0.5*(DX_VAL); \ + double sxmax = (X_VALUE) + 0.5*(DX_VAL); \ + double symin = Xaxis; \ + double symax = (Y_VALUE); \ + /* saturated values for corner coords: */ \ + sxmin = MIN (MAX (sxmin, X0), X1); \ + sxmax = MIN (MAX (sxmax, X0), X1); \ + symin = MAX (MIN (symin, Y0), Y1); \ + symax = MAX (MIN (symax, Y0), Y1); \ + double dy = fabs(symax - symin); \ + double ylow = MAX(symin, symax); \ + DrawRectangle (sxmin, ylow, (DX_VAL), dy); } + +# define HISTOGRAM_OUTFILL(X_VALUE, Y_VALUE, DX_VAL) { \ + /* histogram bar corners */ \ + double sxmin = (X_VALUE) - 0.5*(DX_VAL); \ + double sxmax = (X_VALUE) + 0.5*(DX_VAL); \ + double symin = Xaxis; \ + double symax = (Y_VALUE); \ + /* saturated values for corner coords: */ \ + sxmin = MIN (MAX (sxmin, X0), X1); \ + sxmax = MIN (MAX (sxmax, X0), X1); \ + symin = MAX (MIN (symin, Y0), Y1); \ + symax = MAX (MIN (symax, Y0), Y1); \ + double dy = fabs(symax - symin); \ + double ylow = MAX(symin, symax); \ + FillRectangle (sxmin, ylow, (DX_VAL), dy); \ + fprintf (f, "0.00 0.00 0.00 setrgbcolor\n"); \ + DrawRectangle (sxmin, ylow, (DX_VAL), dy); \ + fprintf (f, "%s setrgbcolor\n", KapaColorRGBString(object->color)); } + +void PSBars (KapaGraphWidget *graph, Gobjects *object, FILE *f, int mode) { + + double mxi = graph[0].axis[0].dfx / (object[0].x1 - object[0].x0); // slope of the x-axis in x-pixels + double mxj = graph[0].axis[1].dfx / (object[0].y1 - object[0].y0); // slope of the x-axis in y-pixels (always 0 for now) + double myi = graph[0].axis[0].dfy / (object[0].x1 - object[0].x0); // slope of the x-axis in x-pixels (always 0 for now) + double myj = graph[0].axis[1].dfy / (object[0].y1 - object[0].y0); // slope of the x-axis in x-pixels + + // intercepts of axes + double bxi = graph[0].axis[0].fx - object[0].x0*graph[0].axis[0].dfx/(object[0].x1 - object[0].x0); + double bxj = -object[0].y0*graph[0].axis[1].dfx/(object[0].y1 - object[0].y0); + double byi = -object[0].x0*graph[0].axis[0].dfy/(object[0].x1 - object[0].x0); + double byj = graph[0].axis[1].fy - object[0].y0*graph[0].axis[1].dfy/(object[0].y1 - object[0].y0); + + double bx = bxi + bxj; + double by = byi + byj; + + // corner coords + double X0 = graph[0].axis[0].fx; + double X1 = graph[0].axis[0].fx + graph[0].axis[0].dfx; + double Y0 = graph[0].axis[1].fy; + double Y1 = graph[0].axis[1].fy + graph[0].axis[1].dfy; + // NOTE: Y0 > Y1 (dfy is negative) + + /* find the first valid datapoint */ + float *x = object[0].x; + float *y = object[0].y; + + /* we are drawing bars which are filled rectangles of height y[i] and width 0.5*dx + one point at a time + + I need to know the distance to the next and prev points to calculate dx + for the first point, dx is x[1] - x[0] + for the last point, dx is x[-1] - x[-2] (x[-1] is the last point) + for the rest, dx is 0.5*(x[i+1] - x[i-1]) + + rather than working out complex on-the-fly logic, I want to find the first and last + valid point in an initial pass, then calculate the above for the remainder. + TBD: make an index vector of only valid points? + */ + + int Ngood = 0; + ALLOCATE_PTR (goodPoint, int, object[0].Npts); + + // x = ??, y = 0: this assumes the xaxis is parallel to the plot window (myi = 0) + // note: Y0 > Y1 : y runs from large on bottom to small on top + float Xaxis = by; + Xaxis = MAX (Y1, MIN (Xaxis, Y0)); + + for (int i = 0; i < object[0].Npts; i++) { if (!(finite(x[i]) && finite(y[i]))) continue; - sx1 = x[i]*mxi + y[i]*mxj + bx; - sy1 = x[i]*myi + y[i]*myj + by; - sx1 = MIN (MAX (sx1, X0), X1); - sy1 = MAX (MIN (sy1, Y0), Y1); - sxa = 0.5*(sx0 + sx1); - DrawLine (sx0, sy0, sxa, sy0); - DrawLine (sxa, sy0, sxa, sy1); - DrawLine (sxa, sy1, sx1, sy1); - sx0 = sx1; sy0 = sy1; - } -# endif + + // coordinate on the screen of the point: + double sx1r = x[i]*mxi + y[i]*mxj + bx; + + if (sx1r < X0) { + // point to the left, skip it + continue; + } + if (sx1r > X1) { + // point to the right, skip it + continue; + } + + goodPoint[Ngood] = i; + Ngood ++; + } + + if (Ngood == 0) { + free (goodPoint); + return; + } + + // if we only have 1 point, draw bar half the width of the screen + // this works fine, but the auto limits for 1 value are somewhat silly + if (Ngood == 1) { + // coordinate on the screen of the point: + int n = goodPoint[0]; + double sx1r = x[n]*mxi + y[n]*mxj + bx; + double sy1r = x[n]*myi + y[n]*myj + by; + + float dx = 0.5*object->size*(X1 - X0); + + switch (mode) { + case KAPA_PLOT_BARS_SOLID: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTLINE: + HISTOGRAM_OUTLINE(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTFILL: + HISTOGRAM_OUTFILL(sx1r, sy1r, dx); + break; + default: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + } + + free (goodPoint); + return; + } + + // first point: + { + int n; + n = goodPoint[1]; + double sx1r = x[n]*mxi + y[n]*mxj + bx; + + n = goodPoint[0]; + double sx0r = x[n]*mxi + y[n]*mxj + bx; + double sy0r = x[n]*myi + y[n]*myj + by; + + double dx = 0.5*object->size*(sx1r - sx0r); + + switch (mode) { + case KAPA_PLOT_BARS_SOLID: + HISTOGRAM_SOLID(sx0r, sy0r, dx); + break; + case KAPA_PLOT_BARS_OUTLINE: + HISTOGRAM_OUTLINE(sx0r, sy0r, dx); + break; + case KAPA_PLOT_BARS_OUTFILL: + HISTOGRAM_OUTFILL(sx0r, sy0r, dx); + break; + default: + HISTOGRAM_SOLID(sx0r, sy0r, dx); + break; + } + } + + for (int i = 1; i < Ngood - 1; i++) { + + int n; + n = goodPoint[i + 1]; + double sx2r = x[n]*mxi + y[n]*mxj + bx; + + n = goodPoint[i - 1]; + double sx0r = x[n]*mxi + y[n]*mxj + bx; + + // below we have 2 factors of 0.5 (we want half of the average spacing) + double dx = 0.5*0.5*object->size*(sx2r - sx0r); + + n = goodPoint[i]; + double sx1r = x[n]*mxi + y[n]*mxj + bx; + double sy1r = x[n]*myi + y[n]*myj + by; + + switch (mode) { + case KAPA_PLOT_BARS_SOLID: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTLINE: + HISTOGRAM_OUTLINE(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTFILL: + HISTOGRAM_OUTFILL(sx1r, sy1r, dx); + break; + default: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + } + } + + // last point: + { + int n; + n = goodPoint[Ngood - 1]; + double sx1r = x[n]*mxi + y[n]*mxj + bx; + double sy1r = x[n]*myi + y[n]*myj + by; + + n = goodPoint[Ngood - 2]; + double sx0r = x[n]*mxi + y[n]*mxj + bx; + + double dx = 0.5*object->size*(sx1r - sx0r); + + switch (mode) { + case KAPA_PLOT_BARS_SOLID: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTLINE: + HISTOGRAM_OUTLINE(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTFILL: + HISTOGRAM_OUTFILL(sx1r, sy1r, dx); + break; + default: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + } + } + free (goodPoint); + return; } @@ -310,6 +551,5 @@ by = byi + byj; - Graphic *graphic = GetGraphic(); - + // scaled colors use the colormap defined for the graphic ALLOCATE (pixel1, float, graphic[0].Npixels); ALLOCATE (pixel2, float, graphic[0].Npixels); @@ -347,5 +587,5 @@ } D = scaleSize ? dz*z[i] : ds; - DrawRectangle (sx, sy, 2*D, 2*D); + DrawRectangle (sx - D, sy - D, 2*D, 2*D); } } @@ -605,5 +845,5 @@ } D = scaleSize ? dz*z[i] : ds; - FillRectangle (sx, sy, 2*D, 2*D); + FillRectangle (sx - D, sy - D, 2*D, 2*D); } } Index: /branches/czw_branch/20170908/Ohana/src/kapa2/src/PSTextlines.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/src/PSTextlines.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/src/PSTextlines.c (revision 40477) @@ -16,6 +16,9 @@ x = graph[0].textline[i].x; y = graphic->dy - graph[0].textline[i].y; + if (graph[0].textline[i].color >= 0) { + fprintf (f, "%s setrgbcolor\n", KapaColorRGBString(graph[0].textline[i].color)); + } SetRotFont (graph[0].textline[i].font, graph[0].textline[i].size); - PSRotText (f, x, y, graph[0].textline[i].text, 5, angle); + PSRotText (f, x, y, graph[0].textline[i].text, graph[0].textline[i].justify, angle); } } Index: /branches/czw_branch/20170908/Ohana/src/kapa2/src/bDrawLabels.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/src/bDrawLabels.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/src/bDrawLabels.c (revision 40477) @@ -40,9 +40,10 @@ for (i = 0; i < graph[0].Ntextline; i++) { if (strcmp (graph[0].textline[i].text, "")) { + bDrawSetColor (buffer, graph[0].textline[i].color); angle = graph[0].textline[i].angle; x = graph[0].textline[i].x; y = graph[0].textline[i].y; SetRotFont (graph[0].textline[i].font, graph[0].textline[i].size); - bDrawRotText (buffer, x, y, graph[0].textline[i].text, 5, angle); + bDrawRotText (buffer, x, y, graph[0].textline[i].text, graph[0].textline[i].justify, angle); } } Index: /branches/czw_branch/20170908/Ohana/src/kapa2/src/bDrawObjects.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/src/bDrawObjects.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/src/bDrawObjects.c (revision 40477) @@ -2,6 +2,6 @@ # define DrawLine(BUF,X1,Y1,X2,Y2) (bDrawLine (BUF, (X1), (Y1), (X2), (Y2))) -# define DrawRectangle(BUF,X,Y,dX,dY) (bDrawRectOpen (BUF, (X-0.5*dX), (Y-0.5*dY), (X+0.5*dX), (Y+0.5*dY))) -# define FillRectangle(BUF,X,Y,dX,dY) (bDrawRectFill (BUF, (X-0.5*dX), (Y-0.5*dY), (X+0.5*dX+1), (Y+0.5*dY+1))) +# define DrawRectangle(BUF,X,Y,dX,dY) (bDrawRectOpen (BUF, (X), (Y), (X+dX), (Y+dY))) +# define FillRectangle(BUF,X,Y,dX,dY) (bDrawRectFill (BUF, (X), (Y), (X+dX+1), (Y+dY+1))) # define DrawCircle(BUF,X1,Y1,R) (bDrawCircle (BUF, (X1), (Y1), (R))) # define FillCircle(BUF,X1,Y1,R) (bDrawCircleFill (BUF, (X1), (Y1), (R))) @@ -9,8 +9,4 @@ # define OpenTriangle(BUF,X1,Y1,X2,Y2,X3,Y3) (bDrawTriOpen (BUF, (X1), (Y1), (X2), (Y2), (X3), (Y3))) -# define CONNECT 0 -# define HISTOGRAM 1 -# define POINTS 2 - static Graphic *graphic; @@ -42,4 +38,13 @@ case KAPA_PLOT_HISTOGRAM: bDrawHistogram (buffer, graph, object); + break; + case KAPA_PLOT_BARS_SOLID: + bDrawBars (buffer, graph, object, KAPA_PLOT_BARS_SOLID); + break; + case KAPA_PLOT_BARS_OUTLINE: + bDrawBars (buffer, graph, object, KAPA_PLOT_BARS_OUTLINE); + break; + case KAPA_PLOT_BARS_OUTFILL: + bDrawBars (buffer, graph, object, KAPA_PLOT_BARS_OUTFILL); break; case KAPA_PLOT_POINTS: @@ -235,25 +240,246 @@ DrawLine (buffer, sx1, sy1, sxa, sy1); DrawLine (buffer, sxa, sy1, sxa, sya); - -# if (0) - sx0 = x[i]*mxi + y[i]*mxj + bx; - sy0 = x[i]*myi + y[i]*myj + by; - sx0 = MIN (MAX (sx0, X0), X1); - sy0 = MAX (MIN (sy0, Y0), Y1); - - /* continue with rest of points */ - for (i++; i < object[0].Npts; i++) { +} + +// uses object->color +# define HISTOGRAM_SOLID(X_VALUE, Y_VALUE, DX_VAL) { \ + /* histogram bar corners */ \ + double sxmin = (X_VALUE) - 0.5*(DX_VAL); \ + double sxmax = (X_VALUE) + 0.5*(DX_VAL); \ + double symin = Xaxis; \ + double symax = (Y_VALUE); \ + /* saturated values for corner coords: */ \ + sxmin = MIN (MAX (sxmin, X0), X1); \ + sxmax = MIN (MAX (sxmax, X0), X1); \ + symin = MAX (MIN (symin, Y0), Y1); \ + symax = MAX (MIN (symax, Y0), Y1); \ + double dy = fabs(symax - symin); \ + double ylow = MIN(symin, symax); \ + FillRectangle (buffer, sxmin, ylow, (DX_VAL), dy); } + +// uses object->color +# define HISTOGRAM_OUTLINE(X_VALUE, Y_VALUE, DX_VAL) { \ + /* histogram bar corners */ \ + double sxmin = (X_VALUE) - 0.5*(DX_VAL); \ + double sxmax = (X_VALUE) + 0.5*(DX_VAL); \ + double symin = Xaxis; \ + double symax = (Y_VALUE); \ + /* saturated values for corner coords: */ \ + sxmin = MIN (MAX (sxmin, X0), X1); \ + sxmax = MIN (MAX (sxmax, X0), X1); \ + symin = MAX (MIN (symin, Y0), Y1); \ + symax = MAX (MIN (symax, Y0), Y1); \ + double dy = fabs(symax - symin); \ + double ylow = MIN(symin, symax); \ + DrawRectangle (buffer, sxmin, ylow, (DX_VAL), dy); } + +# define HISTOGRAM_OUTFILL(X_VALUE, Y_VALUE, DX_VAL) { \ + /* histogram bar corners */ \ + double sxmin = (X_VALUE) - 0.5*(DX_VAL); \ + double sxmax = (X_VALUE) + 0.5*(DX_VAL); \ + double symin = Xaxis; \ + double symax = (Y_VALUE); \ + /* saturated values for corner coords: */ \ + sxmin = MIN (MAX (sxmin, X0), X1); \ + sxmax = MIN (MAX (sxmax, X0), X1); \ + symin = MAX (MIN (symin, Y0), Y1); \ + symax = MAX (MIN (symax, Y0), Y1); \ + double dy = fabs(symax - symin); \ + double ylow = MIN(symin, symax); \ + FillRectangle (buffer, sxmin, ylow, (DX_VAL), dy); \ + bDrawSetColor (buffer, graphic->fore); \ + DrawRectangle (buffer, sxmin, ylow, (DX_VAL), dy); \ + bDrawSetColor (buffer, object[0].color); } + +void bDrawBars (bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object, int mode) { + + double mxi = graph[0].axis[0].dfx / (object[0].x1 - object[0].x0); // slope of the x-axis in x-pixels + double mxj = graph[0].axis[1].dfx / (object[0].y1 - object[0].y0); // slope of the x-axis in y-pixels (always 0 for now) + double myi = graph[0].axis[0].dfy / (object[0].x1 - object[0].x0); // slope of the x-axis in x-pixels (always 0 for now) + double myj = graph[0].axis[1].dfy / (object[0].y1 - object[0].y0); // slope of the x-axis in x-pixels + + // intercepts of axes + double bxi = graph[0].axis[0].fx - object[0].x0*graph[0].axis[0].dfx/(object[0].x1 - object[0].x0); + double bxj = -object[0].y0*graph[0].axis[1].dfx/(object[0].y1 - object[0].y0); + double byi = -object[0].x0*graph[0].axis[0].dfy/(object[0].x1 - object[0].x0); + double byj = graph[0].axis[1].fy - object[0].y0*graph[0].axis[1].dfy/(object[0].y1 - object[0].y0); + + double bx = bxi + bxj; + double by = byi + byj; + + // corner coords + double X0 = graph[0].axis[0].fx; + double X1 = graph[0].axis[0].fx + graph[0].axis[0].dfx; + double Y0 = graph[0].axis[1].fy; + double Y1 = graph[0].axis[1].fy + graph[0].axis[1].dfy; + + /* find the first valid datapoint */ + float *x = object[0].x; + float *y = object[0].y; + + /* we are drawing bars which are filled rectangles of height y[i] and width 0.5*dx + one point at a time + + I need to know the distance to the next and prev points to calculate dx + for the first point, dx is x[1] - x[0] + for the last point, dx is x[-1] - x[-2] (x[-1] is the last point) + for the rest, dx is 0.5*(x[i+1] - x[i-1]) + + rather than working out complex on-the-fly logic, I want to draw the first and last + valid points in an initial pass, then calculate the above for the remainder. + + First, make an index vector of only valid points + */ + + int Ngood = 0; + ALLOCATE_PTR (goodPoint, int, object[0].Npts); + + // x = ??, y = 0: this assumes the xaxis is parallel to the plot window (myi = 0) + // since y runs from large on bottom to small on top, this is slightly backwards + float Xaxis = by; + Xaxis = MAX (Y1, MIN (Xaxis, Y0)); // MIN & MAX reversed for neg y dir + + for (int i = 0; i < object[0].Npts; i++) { if (!(finite(x[i]) && finite(y[i]))) continue; - sx1 = x[i]*mxi + y[i]*mxj + bx; - sy1 = x[i]*myi + y[i]*myj + by; - sx1 = MIN (MAX (sx1, X0), X1); - sy1 = MAX (MIN (sy1, Y0), Y1); - sxa = 0.5*(sx0 + sx1); - DrawLine (buffer, sx0, sy0, sxa, sy0); - DrawLine (buffer, sxa, sy0, sxa, sy1); - DrawLine (buffer, sxa, sy1, sx1, sy1); - sx0 = sx1; sy0 = sy1; - } -# endif + + // coordinate on the screen of the point: + double sx1r = x[i]*mxi + y[i]*mxj + bx; + + if (sx1r < X0) { + // point to the left, skip it + continue; + } + if (sx1r > X1) { + // point to the right, skip it + continue; + } + + goodPoint[Ngood] = i; + Ngood ++; + } + + if (Ngood == 0) { + free (goodPoint); + return; + } + + // if we only have 1 point, draw bar half the width of the screen + // this works fine, but the auto limits for 1 value are somewhat silly + if (Ngood == 1) { + // coordinate on the screen of the point: + int n = goodPoint[0]; + double sx1r = x[n]*mxi + y[n]*mxj + bx; + double sy1r = x[n]*myi + y[n]*myj + by; + + float dx = 0.5*object->size*(X1 - X0); + + switch (mode) { + case KAPA_PLOT_BARS_SOLID: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTLINE: + HISTOGRAM_OUTLINE(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTFILL: + HISTOGRAM_OUTFILL(sx1r, sy1r, dx); + break; + default: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + } + + free (goodPoint); + return; + } + + // first point: + { + int n; + n = goodPoint[1]; + double sx1r = x[n]*mxi + y[n]*mxj + bx; + + n = goodPoint[0]; + double sx0r = x[n]*mxi + y[n]*mxj + bx; + double sy0r = x[n]*myi + y[n]*myj + by; + + double dx = 0.5*object->size*(sx1r - sx0r); + + switch (mode) { + case KAPA_PLOT_BARS_SOLID: + HISTOGRAM_SOLID(sx0r, sy0r, dx); + break; + case KAPA_PLOT_BARS_OUTLINE: + HISTOGRAM_OUTLINE(sx0r, sy0r, dx); + break; + case KAPA_PLOT_BARS_OUTFILL: + HISTOGRAM_OUTFILL(sx0r, sy0r, dx); + break; + default: + HISTOGRAM_SOLID(sx0r, sy0r, dx); + break; + } + } + + for (int i = 1; i < Ngood - 1; i++) { + + int n; + n = goodPoint[i + 1]; + double sx2r = x[n]*mxi + y[n]*mxj + bx; + + n = goodPoint[i - 1]; + double sx0r = x[n]*mxi + y[n]*mxj + bx; + + // below we have 2 factors of 0.5 (we want half of the average spacing) + double dx = 0.5*0.5*object->size*(sx2r - sx0r); + + n = goodPoint[i]; + double sx1r = x[n]*mxi + y[n]*mxj + bx; + double sy1r = x[n]*myi + y[n]*myj + by; + + switch (mode) { + case KAPA_PLOT_BARS_SOLID: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTLINE: + HISTOGRAM_OUTLINE(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTFILL: + HISTOGRAM_OUTFILL(sx1r, sy1r, dx); + break; + default: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + } + } + + // last point: + { + int n; + n = goodPoint[Ngood - 1]; + double sx1r = x[n]*mxi + y[n]*mxj + bx; + double sy1r = x[n]*myi + y[n]*myj + by; + + n = goodPoint[Ngood - 2]; + double sx0r = x[n]*mxi + y[n]*mxj + bx; + + double dx = 0.5*object->size*(sx1r - sx0r); + + switch (mode) { + case KAPA_PLOT_BARS_SOLID: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTLINE: + HISTOGRAM_OUTLINE(sx1r, sy1r, dx); + break; + case KAPA_PLOT_BARS_OUTFILL: + HISTOGRAM_OUTFILL(sx1r, sy1r, dx); + break; + default: + HISTOGRAM_SOLID(sx1r, sy1r, dx); + break; + } + } + free (goodPoint); + return; } @@ -280,6 +506,5 @@ by = byi + byj; - Graphic *graphic = GetGraphic(); - + // scaled colors use the colormap defined for the graphic ALLOCATE (pixel1, unsigned char, graphic[0].Npixels); ALLOCATE (pixel2, unsigned char, graphic[0].Npixels); @@ -293,5 +518,5 @@ } - /**** points are scaled by object.z ***/ + /**** point sizes are scaled by object.size, colors by object.color ***/ int scaleSize = (object[0].size < 0); int scaleColor = (object[0].color < 0); @@ -317,5 +542,5 @@ } D = scaleSize ? dz*z[i] : ds; - DrawRectangle (buffer, sx, sy, 2*D, 2*D); + DrawRectangle (buffer, sx - D, sy - D, 2*D, 2*D); // DrawRectangle takes corner point // plot range saturated by bDrawRectOpen } @@ -611,5 +836,5 @@ } D = scaleSize ? dz*z[i] : ds; - FillRectangle (buffer, sx, sy, 2*D, 2*D); + FillRectangle (buffer, sx - D, sy - D, 2*D, 2*D); // plot range saturated by bDrawRectFill } Index: /branches/czw_branch/20170908/Ohana/src/kapa2/test/textline.sh =================================================================== --- /branches/czw_branch/20170908/Ohana/src/kapa2/test/textline.sh (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/kapa2/test/textline.sh (revision 40477) @@ -0,0 +1,15 @@ + +macro go + dot 0.2 0.2 -sz 4 -pt ocir -c red; textline 0.2 0.2 "test" -justify 0 + dot 0.5 0.2 -sz 4 -pt ocir -c red; textline 0.5 0.2 "test" -justify 1 + dot 0.8 0.2 -sz 4 -pt ocir -c red; textline 0.8 0.2 "test" -justify 2 + dot 0.2 0.5 -sz 4 -pt ocir -c red; textline 0.2 0.5 "test" -justify 3 + dot 0.5 0.5 -sz 4 -pt ocir -c red; textline 0.5 0.5 "test" -justify 4 + dot 0.8 0.5 -sz 4 -pt ocir -c red; textline 0.8 0.5 "test" -justify 5 + dot 0.2 0.8 -sz 4 -pt ocir -c red; textline 0.2 0.8 "test" -justify 6 + dot 0.5 0.8 -sz 4 -pt ocir -c red; textline 0.5 0.8 "test" -justify 7 + dot 0.8 0.8 -sz 4 -pt ocir -c red; textline 0.8 0.8 "test" -justify 8 + + png -name test.png + ps -name test.ps +end Index: /branches/czw_branch/20170908/Ohana/src/libautocode/Makefile.Targets =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/Makefile.Targets (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/Makefile.Targets (revision 40477) @@ -14,4 +14,5 @@ $(ASRC)/average-ps1-v4.$(ARCH).o \ $(ASRC)/average-ps1-v5.$(ARCH).o \ +$(ASRC)/average-ps1-v6.$(ARCH).o \ $(ASRC)/average-ps1-v5-ld.$(ARCH).o \ $(ASRC)/average-ps1-sim.$(ARCH).o \ @@ -29,4 +30,5 @@ $(ASRC)/secfilt-ps1-v4.$(ARCH).o \ $(ASRC)/secfilt-ps1-v5.$(ARCH).o \ +$(ASRC)/secfilt-ps1-v6.$(ARCH).o \ $(ASRC)/secfilt-ps1-v5-ld.$(ARCH).o \ $(ASRC)/secfilt-ps1-sim.$(ARCH).o \ @@ -44,4 +46,5 @@ $(ASRC)/measure-ps1-v4.$(ARCH).o \ $(ASRC)/measure-ps1-v5.$(ARCH).o \ +$(ASRC)/measure-ps1-v6.$(ARCH).o \ $(ASRC)/measure-ps1-v5-ld.$(ARCH).o \ $(ASRC)/measure-ps1-sim.$(ARCH).o \ @@ -53,10 +56,13 @@ $(ASRC)/lensing-ps1-v5-r3.$(ARCH).o \ $(ASRC)/lensing-ps1-v5-ld.$(ARCH).o \ +$(ASRC)/lensing-ps1-v6.$(ARCH).o \ $(ASRC)/lensobj.$(ARCH).o \ $(ASRC)/lensobj-ps1-v5-r0.$(ARCH).o \ $(ASRC)/lensobj-ps1-v5-r1.$(ARCH).o \ $(ASRC)/lensobj-ps1-v5-ld.$(ARCH).o \ +$(ASRC)/lensobj-ps1-v6.$(ARCH).o \ $(ASRC)/starpar.$(ARCH).o \ $(ASRC)/starpar-ps1-v5.$(ARCH).o \ +$(ASRC)/starpar-ps1-v6.$(ARCH).o \ $(ASRC)/starpar-ps1-v5-ld.$(ARCH).o \ $(ASRC)/starpar-ps1-sim.$(ARCH).o \ @@ -65,4 +71,5 @@ $(ASRC)/galphot-ps1-v5-r1.$(ARCH).o \ $(ASRC)/galphot-ps1-v5-ld.$(ARCH).o \ +$(ASRC)/galphot-ps1-v6.$(ARCH).o \ $(ASRC)/missing.$(ARCH).o \ $(ASRC)/photcode.$(ARCH).o \ @@ -76,4 +83,5 @@ $(ASRC)/photcode-ps1-v4.$(ARCH).o \ $(ASRC)/photcode-ps1-v5.$(ARCH).o \ +$(ASRC)/photcode-ps1-v6.$(ARCH).o \ $(ASRC)/photcode-ps1-v5-ld.$(ARCH).o \ $(ASRC)/photcode-ps1-ref.$(ARCH).o \ @@ -90,4 +98,5 @@ $(ASRC)/image-ps1-v4.$(ARCH).o \ $(ASRC)/image-ps1-v5.$(ARCH).o \ +$(ASRC)/image-ps1-v6.$(ARCH).o \ $(ASRC)/image-ps1-v5-ld.$(ARCH).o \ $(ASRC)/image-ps1-sim.$(ARCH).o \ @@ -136,4 +145,5 @@ $(AINC)/average-ps1-v4.h \ $(AINC)/average-ps1-v5.h \ +$(AINC)/average-ps1-v6.h \ $(AINC)/average-ps1-v5-ld.h \ $(AINC)/average-ps1-sim.h \ @@ -151,4 +161,5 @@ $(AINC)/secfilt-ps1-v4.h \ $(AINC)/secfilt-ps1-v5.h \ +$(AINC)/secfilt-ps1-v6.h \ $(AINC)/secfilt-ps1-v5-ld.h \ $(AINC)/secfilt-ps1-sim.h \ @@ -166,4 +177,5 @@ $(AINC)/measure-ps1-v4.h \ $(AINC)/measure-ps1-v5.h \ +$(AINC)/measure-ps1-v6.h \ $(AINC)/measure-ps1-v5-ld.h \ $(AINC)/measure-ps1-sim.h \ @@ -175,10 +187,13 @@ $(AINC)/lensing-ps1-v5-r3.h \ $(AINC)/lensing-ps1-v5-ld.h \ +$(AINC)/lensing-ps1-v6.h \ $(AINC)/lensobj.h \ $(AINC)/lensobj-ps1-v5-r0.h \ $(AINC)/lensobj-ps1-v5-r1.h \ $(AINC)/lensobj-ps1-v5-ld.h \ +$(AINC)/lensobj-ps1-v6.h \ $(AINC)/starpar.h \ $(AINC)/starpar-ps1-v5.h \ +$(AINC)/starpar-ps1-v6.h \ $(AINC)/starpar-ps1-v5-ld.h \ $(AINC)/starpar-ps1-sim.h \ @@ -187,4 +202,5 @@ $(AINC)/galphot-ps1-v5-r1.h \ $(AINC)/galphot-ps1-v5-ld.h \ +$(AINC)/galphot-ps1-v6.h \ $(AINC)/missing.h \ $(AINC)/photcode.h \ @@ -198,4 +214,5 @@ $(AINC)/photcode-ps1-v4.h \ $(AINC)/photcode-ps1-v5.h \ +$(AINC)/photcode-ps1-v6.h \ $(AINC)/photcode-ps1-v5-ld.h \ $(AINC)/photcode-ps1-ref.h \ @@ -212,4 +229,5 @@ $(AINC)/image-ps1-v4.h \ $(AINC)/image-ps1-v5.h \ +$(AINC)/image-ps1-v6.h \ $(AINC)/image-ps1-v5-ld.h \ $(AINC)/image-ps1-sim.h \ Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/average-ps1-v6.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/average-ps1-v6.d (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/average-ps1-v6.d (revision 40477) @@ -0,0 +1,92 @@ +STRUCT Average_PS1_V6 +EXTNAME DVO_AVERAGE_PS1_V6 +TYPE BINTABLE +SIZE 192 +DESCRIPTION DVO Average Object Table + +# elements of data structure / FITS table + +FIELD R, RA, double, RA, decimal degrees +FIELD D, DEC, double, DEC, decimal degrees +FIELD dR, RA_ERR, float, RA error, arcsec +FIELD dD, DEC_ERR, float, DEC error, arcsec + +FIELD uR, U_RA, float, RA*cos(D) proper-motion, arcsec/year +FIELD uD, U_DEC, float, DEC proper-motion, arcsec/year +FIELD duR, V_RA_ERR, float, RA*cos(D) p-m error, arcsec/year +FIELD duD, V_DEC_ERR, float, DEC p-m error, arcsec/year +FIELD P, PAR, float, parallax, arcsec +FIELD dP, PAR_ERR, float, parallax error, arcsec + +FIELD Rstk, RA_STK, double, RA on stack, decimal degrees +FIELD Dstk, DEC_STK, double, DEC on stack, decimal degrees +FIELD dRstk, RA_STK_ERR, float, RA error on stack, arcsec +FIELD dDstk, DEC_STK_ERR, float, DEC error on stack, arcsec + +FIELD ChiSqAve, CHISQ_POS, float, astrometry analysis chisq +FIELD ChiSqPM, CHISQ_PM, float, astrometry analysis chisq +FIELD ChiSqPar, CHISQ_PLX, float, astrometry analysis chisq +FIELD Tmean, MEAN_EPOCH, int, mean epoch (PM,PAR ref), unix time seconds +FIELD Trange, TIME_RANGE, int, mean epoch (PM,PAR ref), unix time seconds + +FIELD psfQF, PSF_QF, float, psf coverage (bad masks) +FIELD psfQFperf, PSF_QF_PERF, float, psf coverage (all masks) + +FIELD stargal, STARGAL_SEP, float, star / galaxy separator, 1/100 arcsec +FIELD Npos, NUMBER_POS, unsigned short, number of detections used for astrometry + +# this limits us to a max of 64k measurements per object +FIELD Nmeasure, NMEASURE, unsigned short, number of psf measurements +FIELD Nmissing, NMISSING, unsigned short, number of missings +FIELD Nlensing, NLENSING, unsigned short, number of lensing measurements +FIELD Nlensobj, NLENSOBJ, unsigned short, number of lensing measurements +FIELD Ngalphot, NGALPHOT, unsigned short, number of galphot measurements + +FIELD measureOffset, OFF_MEASURE, int, offset to first psf measurement +FIELD missingOffset, OFF_MISSING, int, offset to first missing obs +FIELD lensingOffset, OFF_LENSING, int, offset to first lensing obs +FIELD lensobjOffset, OFF_LENSOBJ, int, offset to mean lensing data +FIELD starparOffset, OFF_STARPAR, int, offset to stellar parameter data +FIELD galphotOffset, OFF_GALPHOT, int, offset to galphot object entry + +FIELD refColorBlue, REF_COLOR_BLUE, float, color of astrometry ref stars +FIELD refColorRed, REF_COLOR_RED, float, color of astrometry ref stars + +FIELD tessID, TESS_ID, char, ID of tessellation for primary skycell +FIELD skycellID, SKYCELL_ID, char, ID of skycell for primary skycell +FIELD projectionID, PROJECTION_ID, short, ID of projection for primary skycell + +FIELD Nstarpar, NSTARPAR, short, number of stellar parameter entries +FIELD NwarpOK, NWARP_OK, short, total number of warp measurements with psf_qf > 0.0 + +# 'flags' was called 'code' prior to 2009.02.07 +FIELD flags, FLAGS, uint32_t, average object flags (star; ghost; etc) +FIELD photFlagsUpper, PHOTFLAGS_U, uint32_t, upper bit of 2 bit summary of per-measure photflags +FIELD photFlagsLower, PHOTFLAGS_L, uint32_t, lower bit of 2 bit summary of per-measure photflags + +# objID + catID gives a unique ID for all objects in the database +FIELD objID, OBJ_ID, unsigned int, unique ID for object in table +FIELD catID, CAT_ID, unsigned int, unique ID for table in which object was first realized +FIELD extID, EXT_ID, uint64_t, external ID for object (eg PSPS objID) + +# replace extIDgc (unused) with uRgal, uDgal: +# FIELD extIDgc, EXT_ID_GC, uint64_t, external ID for object in galactic coords +FIELD uRgal, U_RA_GAL, float, modeled proper motion based on galactic motion +FIELD uDgal, U_DEC_GAL, float, modeled proper motion based on galactic motion + +# 4 double, 2 uint64_t, 17 float, 12 int, 6 short +# 4*8 + 2*8 + 17*4 + 12*4 + 6*2 = 176 +# for 5G objects, expect 840G + +# photflagsUpper & photflagsLower: we have Nmeasures of a given source +# using psphot. each of these as a photFlag field, with bits +# describing the detection quality. photflagsUpper and photFlagsLower +# represent a 2 bit value (0-3) which defines the frequency of a given +# bit in the measurements: +# for a given bit, if that bit is raised for these percentile ranges, +# the following bits are set: +# min | max | L | U +# 0% | 25% | 0 | 0 +# 25% | 50% | 1 | 0 +# 50% | 75% | 0 | 1 +# 75% | 100% | 1 | 1 Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/convert.sh =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/convert.sh (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/convert.sh (revision 40477) @@ -0,0 +1,7 @@ +#!/bin/csh -f + +foreach f ( *-ps1-v5.d ) + set g = `echo $f | sed s/ps1-v5/ps1-v6/` + cat $f | sed s/PS1_V5/PS1_V6/ > $g +end + Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/galphot-ps1-v6.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/galphot-ps1-v6.d (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/galphot-ps1-v6.d (revision 40477) @@ -0,0 +1,31 @@ +STRUCT GalPhot_PS1_V6 +EXTNAME DVO_GALPHOT_PS1_V6 +TYPE BINTABLE +SIZE 80 +DESCRIPTION DVO Galaxy Shape Table + +FIELD Xfit, XFIT, float, centroid for fit +FIELD Yfit, YFIT, float, centroid for fit +FIELD mag, MAG, float, galaxy magnitude +FIELD magErr, MAG_ERR, float, galaxy magnitude error +FIELD majorAxis, MAJOR_AXIS, float, major axis size +FIELD minorAxis, MINOR_AXIS, float, minor axis size +FIELD majorAxisErr, MAJOR_AXIS_ERR, float, major axis size error +FIELD minorAxisErr, MINOR_AXIS_ERR, float, minor axis size error +FIELD theta, THETA, float, angle +FIELD thetaErr, THETA_ERR, float, angle error +FIELD index, INDEX, float, sersic index (if relevant) +FIELD chisq, CHISQ, float, fit chisq + +FIELD Npix, NPIX, float, fitted pixels + +FIELD objID, OBJ_ID, unsigned int, unique ID for object in table +FIELD catID, CAT_ID, unsigned int, unique ID for table in which object was first realized +FIELD detID, DET_ID, unsigned int, detection ID +FIELD imageID, IMAGE_ID, unsigned int, reference to DVO image ID + +FIELD averef, AVEREF, unsigned int, reference to average table +FIELD flags, FLAGS, unsigned int, info flags + +FIELD photcode, PHOTCODE, short +FIELD modelType, MODEL_TYPE, short, mean lensing smear object E2 Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/image-ps1-v6.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/image-ps1-v6.d (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/image-ps1-v6.d (revision 40477) @@ -0,0 +1,80 @@ +STRUCT Image_PS1_V6 +EXTNAME DVO_IMAGE_PS1_V6 +TYPE BINTABLE +SIZE 368 +DESCRIPTION DVO Image Table + +# elements of the image structure +# careful of 8-byte boundaries + +SUBSTRUCT coords, COORDS, CoordsDisk, astrometric data +SUBFIELD crval1, CRVAL1, double, coordinate at reference pixel +SUBFIELD crval2, CRVAL2, double, coordinate at reference pixel +SUBFIELD crpix1, CRPIX1, float, coordinate of reference pixel +SUBFIELD crpix2, CRPIX2, float, coordinate of reference pixel +SUBFIELD cdelt1, CDELT1, float, degrees per pixel +SUBFIELD cdelt2, CDELT2, float, degrees per pixel +SUBFIELD pc1_1, PC1_1, float, rotation matrix +SUBFIELD pc1_2, PC1_2, float, rotation matrix +SUBFIELD pc2_1, PC2_1, float, rotation matrix +SUBFIELD pc2_2, PC2_2, float, rotation matrix +SUBFIELD polyterms, POLYTERMS, float[7][2], higher order warping terms +SUBFIELD ctype, CTYPE, char[15], coordinate type +SUBFIELD Npolyterms, NPOLYTERMS, char, order of polynomial +# 120 bytes + +# change this to a double mjd? +FIELD tzero, TZERO, e_time, readout time (row 0) +FIELD nstar, NSTAR, unsigned int, number of stars on image +FIELD secz, SECZ, float, airmass, mag +FIELD NX, NX, unsigned short, image width +FIELD NY, NY, unsigned short, image height +FIELD apmifit, APMIFIT, float, aperture correction, mag +FIELD dapmifit, DAPMIFIT, float, apmifit error, mag +FIELD McalPSF, MCAL_PSF, float, calibration mag, mag +FIELD McalAPER, MCAL_APER, float, calibration mag for Aperture, mag +FIELD dMcal, DMCAL, float, error on Mcal, mag +FIELD McalChiSq, XM, float, image chisq +FIELD padding_1, PADDING, short, identifier for CCD, +FIELD photcode, PHOTCODE, short, identifier for CCD, +FIELD exptime, EXPTIME, float, exposure time, seconds +FIELD sidtime, ST, float, sidereal time of exposure +FIELD latitude, LAT, float, observatory latitude, degrees +# 56 bytes + +FIELD RAo, RA_CENTER, float, image center, degrees +FIELD DECo, DEC_CENTER, float, image center, degrees +FIELD Radius, RADIUS, float, image radius, degrees +FIELD refColorBlue, REF_COLOR_BLUE, float, median astrometry ref color +FIELD refColorRed, REF_COLOR_RED, float, median astrometry ref color + +# should we define the max length of name as a macro? +FIELD name, NAME, char[117], name of original image +FIELD detection_limit, DETECTION_LIMIT, unsigned char, detection limit, 10*mag +FIELD saturation_limit, SATURATION_LIMIT, unsigned char, saturation limit, 10*mag +FIELD cerror, CERROR, unsigned char, astrometric error, 50*arcsec +FIELD fwhm_x, FWHM_X, unsigned char, PSF x width, 25*arcsec +FIELD fwhm_y, FWHM_Y, unsigned char, PSF y width, 25*arcsec +FIELD trate, TRATE, unsigned char, scan rate, 100 usec/pixel +FIELD ccdnum, CCDNUM, unsigned char, CCD ID number +FIELD flags, FLAGS, unsigned int, image quality flags +FIELD imageID, IMAGE_ID, unsigned int, internal image ID +FIELD parentID, PARENT_ID, unsigned int, associated ref image +FIELD externID, EXTERN_ID, unsigned int, external image ID +FIELD sourceID, SOURCE_ID, unsigned short, analysis source ID +# 48 bytes + +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD ubercalDist, UBERCAL_DIST, short, distance to nearest ubercal image + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/image.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/image.d (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/image.d (revision 40477) @@ -2,6 +2,9 @@ EXTNAME DVO_IMAGE TYPE BINTABLE -SIZE 384 +SIZE 392 DESCRIPTION DVO Image Table + +## XXX NOTE: as of 2015.01.11 (r 37807), I had to move this structure from the autocode +## version to libdvo/include/libdvo_astro.h to support self-references # elements of the image structure @@ -35,12 +38,14 @@ FIELD apmifit, APMIFIT, float, aperture correction, mag FIELD dapmifit, DAPMIFIT, float, apmifit error, mag -FIELD Mcal, MCAL, float, calibration mag, mag +FIELD McalPSF, MCAL_PSF, float, calibration mag for PSF, mag +FIELD McalAPER, MCAL_APER, float, calibration mag for Aperture, mag FIELD dMcal, DMCAL, float, error on Mcal, mag -FIELD Xm, XM, short, image chisq, 10*log(value) +FIELD McalChiSq, XM, float, image chisq +FIELD padding_1, PADDING, short, identifier for CCD, FIELD photcode, PHOTCODE, short, identifier for CCD, FIELD exptime, EXPTIME, float, exposure time, seconds FIELD sidtime, ST, float, sidereal time of exposure FIELD latitude, LAT, float, observatory latitude, degrees -# 40 bytes +# 56 bytes FIELD RAo, RA_CENTER, float, image center, degrees Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/lensing-ps1-v6.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/lensing-ps1-v6.d (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/lensing-ps1-v6.d (revision 40477) @@ -0,0 +1,60 @@ +STRUCT Lensing_PS1_V6 +EXTNAME DVO_LENSING_PS1_V6 +TYPE BINTABLE +SIZE 160 +DESCRIPTION DVO Lensing Table + +FIELD X11_sm_obj, X11_SM_OBJ, float, lensing smear object X11 +FIELD X12_sm_obj, X12_SM_OBJ, float, lensing smear object X12 +FIELD X22_sm_obj, X22_SM_OBJ, float, lensing smear object X22 +FIELD E1_sm_obj, E1_SM_OBJ, float, lensing smear object E1 +FIELD E2_sm_obj, E2_SM_OBJ, float, lensing smear object E2 + +FIELD X11_sh_obj, X11_SH_OBJ, float, lensing shear object X11 +FIELD X12_sh_obj, X12_SH_OBJ, float, lensing shear object X12 +FIELD X22_sh_obj, X22_SH_OBJ, float, lensing shear object X22 +FIELD E1_sh_obj, E1_SH_OBJ, float, lensing shear object E1 +FIELD E2_sh_obj, E2_SH_OBJ, float, lensing shear object E2 + +FIELD X11_sm_psf, X11_SM_PSF, float, lensing smear psf stars X11 +FIELD X12_sm_psf, X12_SM_PSF, float, lensing smear psf stars X12 +FIELD X22_sm_psf, X22_SM_PSF, float, lensing smear psf stars X22 +FIELD E1_sm_psf, E1_SM_PSF, float, lensing smear psf stars E1 +FIELD E2_sm_psf, E2_SM_PSF, float, lensing smear psf stars E2 + +FIELD X11_sh_psf, X11_SH_PSF, float, lensing shear psf stars X11 +FIELD X12_sh_psf, X12_SH_PSF, float, lensing shear psf stars X12 +FIELD X22_sh_psf, X22_SH_PSF, float, lensing shear psf stars X22 +FIELD E1_sh_psf, E1_SH_PSF, float, lensing shear psf stars E1 +FIELD E2_sh_psf, E2_SH_PSF, float, lensing shear psf stars E2 + +FIELD E1_psf, E1_PSF, float, polarization psf stars E1 +FIELD E2_psf, E2_PSF, float, polarization psf stars E2 + +FIELD F_ApR5, FLUX_AP_R5, float, Flux inside r = 5 +FIELD dF_ApR5, FLUX_ERR_AP_R5, float, error on Flux inside r = 5 +FIELD sF_ApR5, FLUX_STD_AP_R5, float, stdev of Flux inside r = 5 +FIELD fF_ApR5, FLUX_FIL_AP_R5, float, fill factor for Flux inside r = 5 + +FIELD F_ApR6, FLUX_AP_R6, float, Flux inside r = 6 +FIELD dF_ApR6, FLUX_ERR_AP_R6, float, error on Flux inside r = 6 +FIELD sF_ApR6, FLUX_STD_AP_R6, float, stdev of Flux inside r = 6 +FIELD fF_ApR6, FLUX_FIL_AP_R6, float, fill factor for Flux inside r = 6 + +FIELD F_ApR7, FLUX_AP_R7, float, Flux inside r = 7 +FIELD dF_ApR7, FLUX_ERR_AP_R7, float, error on Flux inside r = 7 +FIELD sF_ApR7, FLUX_STD_AP_R7, float, stdev of Flux inside r = 7 +FIELD fF_ApR7, FLUX_FIL_AP_R7, float, fill factor for Flux inside r = 7 + +FIELD detID, DET_ID, unsigned int, detection ID +FIELD objID, OBJ_ID, unsigned int, unique ID for object in table +FIELD catID, CAT_ID, unsigned int, unique ID for table in which object was first realized +FIELD averef, AVE_REF, unsigned int, reference to average entry + +FIELD imageID, IMAGE_ID, unsigned int, reference to DVO image ID +FIELD oldImID, OLD_IM_ID, int, saved reference to (wrong) DVO image ID + +# 28 x float +# 4 x int +# = 128 bytes / detection +# for 2.5G objects + 60 overlaps, expect 15TB for forced warp Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/lensobj-ps1-v6.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/lensobj-ps1-v6.d (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/lensobj-ps1-v6.d (revision 40477) @@ -0,0 +1,58 @@ +STRUCT Lensobj_PS1_V6 +EXTNAME DVO_LENSOBJ_PS1_V6 +TYPE BINTABLE +SIZE 152 +DESCRIPTION DVO Lensobj Table + +FIELD X11_sm_obj, X11_SM_OBJ, float, mean lensing smear object X11 +FIELD X12_sm_obj, X12_SM_OBJ, float, mean lensing smear object X12 +FIELD X22_sm_obj, X22_SM_OBJ, float, mean lensing smear object X22 +FIELD E1_sm_obj, E1_SM_OBJ, float, mean lensing smear object E1 +FIELD E2_sm_obj, E2_SM_OBJ, float, mean lensing smear object E2 + +FIELD X11_sh_obj, X11_SH_OBJ, float, mean lensing shear object X11 +FIELD X12_sh_obj, X12_SH_OBJ, float, mean lensing shear object X12 +FIELD X22_sh_obj, X22_SH_OBJ, float, mean lensing shear object X22 +FIELD E1_sh_obj, E1_SH_OBJ, float, mean lensing shear object E1 +FIELD E2_sh_obj, E2_SH_OBJ, float, mean lensing shear object E2 + +FIELD X11_sm_psf, X11_SM_PSF, float, mean lensing smear psf stars X11 +FIELD X12_sm_psf, X12_SM_PSF, float, mean lensing smear psf stars X12 +FIELD X22_sm_psf, X22_SM_PSF, float, mean lensing smear psf stars X22 +FIELD E1_sm_psf, E1_SM_PSF, float, mean lensing smear psf stars E1 +FIELD E2_sm_psf, E2_SM_PSF, float, mean lensing smear psf stars E2 + +FIELD X11_sh_psf, X11_SH_PSF, float, mean lensing shear psf stars X11 +FIELD X12_sh_psf, X12_SH_PSF, float, mean lensing shear psf stars X12 +FIELD X22_sh_psf, X22_SH_PSF, float, mean lensing shear psf stars X22 +FIELD E1_sh_psf, E1_SH_PSF, float, mean lensing shear psf stars E1 +FIELD E2_sh_psf, E2_SH_PSF, float, mean lensing shear psf stars E2 + +FIELD F_ApR5, FLUX_AP_R5, float, Flux inside r = 5 +FIELD dF_ApR5, FLUX_ERR_AP_R5, float, error on Flux inside r = 5 +FIELD sF_ApR5, FLUX_STD_AP_R5, float, stdev of Flux inside r = 5 +FIELD fF_ApR5, FLUX_FIL_AP_R5, float, fill factor for Flux inside r = 5 + +FIELD F_ApR6, FLUX_AP_R6, float, Flux inside r = 6 +FIELD dF_ApR6, FLUX_ERR_AP_R6, float, error on Flux inside r = 6 +FIELD sF_ApR6, FLUX_STD_AP_R6, float, stdev of Flux inside r = 6 +FIELD fF_ApR6, FLUX_FIL_AP_R6, float, fill factor for Flux inside r = 6 + +FIELD F_ApR7, FLUX_AP_R7, float, Flux inside r = 7 +FIELD dF_ApR7, FLUX_ERR_AP_R7, float, error on Flux inside r = 7 +FIELD sF_ApR7, FLUX_STD_AP_R7, float, stdev of Flux inside r = 7 +FIELD fF_ApR7, FLUX_FIL_AP_R7, float, fill factor for Flux inside r = 7 + +FIELD gamma, GAMMA, float +FIELD E1, E1, float +FIELD E2, E2, float + +FIELD objID, OBJ_ID, unsigned int, unique ID for object in table +FIELD catID, CAT_ID, unsigned int, unique ID for table in which object was first realized +FIELD photcode, PHOTCODE, short +FIELD Nmeas, NMEAS, short + +# 31 x float +# 3 x int +# = 136 bytes / detection +# for 2.5G objects + 60 overlaps, expect 15TB for forced warp Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/measure-ps1-v6.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/measure-ps1-v6.d (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/measure-ps1-v6.d (revision 40477) @@ -0,0 +1,111 @@ +STRUCT Measure_PS1_V6 +EXTNAME DVO_MEASURE_PS1_V6 +TYPE BINTABLE +SIZE 240 +DESCRIPTION DVO Detection Measurement Table + +FIELD R, RA, double, RA at epoch, degrees +FIELD D, DEC, double, DEC at epoch, degrees +FIELD M, MAG, float, catalog mag, mag +FIELD dM, MAG_ERR, float, mag error, mag +FIELD Map, M_APER, float, aperture mag, mag +FIELD dMap, M_APER_ERR, float, aperture mag, mag +FIELD Mkron, M_KRON, float, kron magnitude, mag +FIELD dMkron, M_KRON_ERR, float, kron magnitude error, mag +FIELD McalPSF, MCAL_PSF, float, image cal mag, mag +FIELD dMcal, MAG_CAL_ERR, float, systematic calibration error, mag +FIELD dt, M_TIME, float, exposure time, 2.5*log(exptime) + +# needed for stacks, forced warp, and diff +FIELD FluxPSF, FLUX_PSF, float, flux from psf fit, counts/sec +FIELD dFluxPSF, FLUX_PSF_ERR, float, error on psf flux, counts/sec +FIELD FluxKron, FLUX_KRON, float, flux from kron ap, counts/sec +FIELD dFluxKron, FLUX_KRON_ERR, float, error on kron flux, counts/sec +FIELD FluxAp, FLUX_AP, float, flux from ap ap, counts/sec +FIELD dFluxAp, FLUX_AP_ERR, float, error on ap flux, counts/sec + +# note that with airmass = 1.0 / cos(90 - alt), we have full alt/az representation +FIELD airmass, AIRMASS, float, (airmass - 1), airmass +FIELD az, AZ, float, telescope azimuth + +# new field elements needed for Pan-STARRS: +FIELD Xccd, X_CCD, float, X coord on chip (raw value), pixels +FIELD Yccd, Y_CCD, float, Y coord on chip (raw value), pixels + +FIELD Xfix, X_FIX, float, X coord after correction, pixels +FIELD Yfix, Y_FIX, float, Y coord after correction, pixels + +FIELD XoffKH, X_OFF_KH, float, X offset from correction, pixels +FIELD YoffKH, Y_OFF_KH, float, Y offset from correction, pixels +FIELD XoffDCR, X_OFF_DCR, float, X offset from correction, pixels +FIELD YoffDCR, Y_OFF_DCR, float, Y offset from correction, pixels +FIELD XoffCAM, X_OFF_CAM, float, X offset from correction, pixels +FIELD YoffCAM, Y_OFF_CAM, float, Y offset from correction, pixels + +FIELD Mflat, M_FLAT, float, Static Flat-field offset, mag +FIELD McalAPER, MCAL_APER, float, image cal mag for apertures, mag + +# could these be packed into fewer bits? +FIELD Sky, SKY_FLUX, float, local estimate of sky flux, counts/sec +FIELD dSky, SKY_FLUX_ERR, float, local estimate of sky flux, counts/sec + +FIELD t, TIME, int, time in seconds (UNIX) +FIELD averef, AVE_REF, unsigned int, reference to average entry + +# internally, this is an unsigned int; however, we do NOT convert with TZERO/TSCAL on output +FIELD detID, DET_ID, unsigned int, detection ID +FIELD objID, OBJ_ID, unsigned int, unique ID for object in table +FIELD catID, CAT_ID, unsigned int, unique ID for table in which object was first realized + +# extID needs to be on an 8-byte boundary +# PSPS uses a 64-bit detection ID +FIELD extID, EXT_ID, uint64_t, external ID (eg PSPS detID) + +FIELD imageID, IMAGE_ID, unsigned int, reference to DVO image ID + +# do we need more resolution than a short? should this be a log? +FIELD psfQF, PSF_QF, float, psf coverage/quality factor +FIELD psfQFperf, PSF_QF_PEFECT, float, psf coverage / quality factor (all mask bits) +FIELD psfChisq, PSF_CHISQ, float, psf fit chisq + +FIELD psfNdof, PSF_NDOF, int, psf degrees of freedom +FIELD psfNpix, PSF_NPIX, int, psf number of pixels +FIELD photFlags2, PHOT_FLAGS2, int, flags supplied by photometry program +FIELD extNsigma, EXT_NSIGMA, float, Nsigma deviation towards EXT + +# model shape parameters +FIELD FWx, FWHM_MAJOR, short, object fwhm major axis, 1/100 of pixels +FIELD FWy, FWHM_MINOR, short, object fwhm minor axis, 1/100 of pixels +FIELD theta, PSF_THETA, short, angle wrt ccd X dir, (0xffff/360) deg + +# moments +FIELD Mxx, MXX, short, second moments in pixel coords, 1/100 of pixels +FIELD Mxy, MXY, short, second moments in pixel coords, 1/100 of pixels +FIELD Myy, MYY, short, second moments in pixel coords, 1/100 of pixels + +# fractional exposure time +FIELD t_msec, TIME_MSEC, unsigned short, time fraction of second, milliseconds +FIELD photcode, PHOTCODE, unsigned short, photcode + +# position errors +FIELD dXccd, X_CCD_ERR, short, X coord error on chip, 1/100 of pixels +FIELD dYccd, Y_CCD_ERR, short, Y coord error on chip, 1/100 of pixels +FIELD dRsys, POS_SYS_ERR, short, systematic error from astrom, 1/100 of pixels + +# local astrometry scales +FIELD posangle, POSANGLE, short, position angle sky to chip, (0xffff/360) deg +FIELD pltscale, PLTSCALE, float, plate scale, arcsec/pixel + +FIELD dbFlags, DB_FLAGS, unsigned int, flags supplied by analysis in database +FIELD photFlags, PHOT_FLAGS, unsigned int, flags supplied by photometry program + +FIELD padding, PADDING, int, padding to ensure 8byte blocks + +# 2 x double +# 35 x float +# 11 x int or unsigned int +# 12 x short or unsigned short +# 1 x uint64_t +# = 232 bytes / detection +# for 2.5G objects * 60 overlaps, expect 35TB for forced warp +# for 120G detections, expect 28TB Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/measure.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/measure.d (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/measure.d (revision 40477) @@ -13,5 +13,5 @@ FIELD Mkron, M_KRON, float, kron magnitude, mag FIELD dMkron, M_KRON_ERR, float, kron magnitude error, mag -FIELD Mcal, M_CAL, float, image cal mag, mag +FIELD McalPSF, MCAL_PSF, float, image cal mag, mag FIELD dMcal, MAG_CAL_ERR, float, systematic calibration error, mag FIELD dt, M_TIME, float, exposure time, 2.5*log(exptime) @@ -44,5 +44,5 @@ FIELD Mflat, M_FLAT, float, Static Flat-field offset, mag -FIELD dummy2, PADDING, int, unused 4 bytes +FIELD McalAPER, MCAL_APER, float, image cal mag for apertures, mag # could these be packed into fewer bits? @@ -53,8 +53,10 @@ FIELD averef, AVE_REF, unsigned int, reference to average entry +# internally, this is an unsigned int; however, we do NOT convert with TZERO/TSCAL on output FIELD detID, DET_ID, unsigned int, detection ID FIELD objID, OBJ_ID, unsigned int, unique ID for object in table FIELD catID, CAT_ID, unsigned int, unique ID for table in which object was first realized +# extID needs to be on an 8-byte boundary # PSPS uses a 64-bit detection ID FIELD extID, EXT_ID, uint64_t, external ID (eg PSPS detID) @@ -91,4 +93,5 @@ FIELD dRsys, POS_SYS_ERR, short, systematic error from astrom, 1/100 of pixels +# local astrometry scales FIELD posangle, POSANGLE, short, position angle sky to chip, (0xffff/360) deg FIELD pltscale, PLTSCALE, float, plate scale, arcsec/pixel Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/photcode-ps1-v6.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/photcode-ps1-v6.d (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/photcode-ps1-v6.d (revision 40477) @@ -0,0 +1,33 @@ +STRUCT PhotCode_PS1_V6 +EXTNAME DVO_PHOTCODE_PS1_V6 +TYPE BINTABLE +SIZE 112 +DESCRIPTION DVO Photcode Description Table + +# elements of data structure / FITS table +FIELD code, CODE, unsigned short, code number (stored in Measure.source) +FIELD name, NAME, char[32], name for filter combination +FIELD type, TYPE, char, PRI/SEC/DEP/REF +FIELD dummy, DUMMY, char[3], padding +FIELD C, C_LAM, short, primary phot calibration terms (millimags) +FIELD dC, C_LAM_ERR, short, primary phot calibration terms (millimags) +FIELD dX, X_ERR, short, primary phot calibration terms (millimags) +FIELD K, K, float, secondary phot calibration terms (millimags) +FIELD c1, C1, int, color is average.M[c1] - average.M[c2] +FIELD c2, C2, int, color is average.M[c1] - average.M[c2] +FIELD equiv, EQUIV, int, this dependent filter is equivalent to equiv PRI/SEC +FIELD Nc, NC, int, number of color terms +FIELD X, X, float[4], color terms $X[0]*mc + X[1]*mc^2 + X[2]*mc^3$, etc +FIELD astromErrSys, ASTROM_ERR_SYS, float, systematic astrometry error (arcsec) +FIELD astromErrScale, ASTROM_ERR_SCALE, float, astrometric error scale +FIELD astromErrMagScale, ASTROM_ERR_MAG_SCALE, float, astrometric error / mag error scale +FIELD astromPoorMask, ASTROM_POOR_MASK, unsigned int, detections matching this mask should only be used in emergencies +FIELD astromBadMask, ASTROM_BAD_MASK, unsigned int, detections matching this mask should not be used +FIELD photomErrSys, PHOTOM_ERR_SYS, float, systematic photometric error +FIELD photomPoorMask, PHOTOM_POOR_MASK, unsigned int, detections matching this mask should only be used in emergencies +FIELD photomBadMask, PHOTOM_BAD_MASK, unsigned int, detections matching this mask should not be used + +# dR_total^2 = dR_sys^2 + AS * dR_obs^2 + MS * dM_obs^2 +# dR_sys : systematicAstrometryError +# AS : astrometryErrorScale +# MS : astrometryErrorMagScale Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/secfilt-ps1-v6.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/secfilt-ps1-v6.d (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/secfilt-ps1-v6.d (revision 40477) @@ -0,0 +1,81 @@ +STRUCT SecFilt_PS1_V6 +EXTNAME DVO_SECFILT_PS1_V6 +TYPE BINTABLE +SIZE 176 +DESCRIPTION DVO SecFilt : Secondary Filter Data + +## *** this section is for per-exposure mean values *** (unlabled values are implicitly PSF values) +FIELD M, MAG, float, average mag in this band, mags +FIELD dM, MAG_ERR, float, formal error on average mag, mags +FIELD Map, MAG_AP, float, ave aperture mag in this band, mags +FIELD dMap, MAG_AP_ERR, float, ave aperture mag in this band, mags +FIELD sMap, MAG_AP_STDEV, float, standard deviation of ap mags, mags +FIELD Mkron, MAG_KRON, float, ave kron mag in this band, mags +FIELD dMkron, MAG_KRON_ERR, float, formal error on average kron mag, mags +FIELD sMkron, MAG_KRON_STDEV, float, standard deviation of kron mags, mags + +# XXX I could add these fields to secfilt or calculate in dvopsps? +FIELD psfQfMax, PSF_QF_MAX, float, best psfQf for this filter +FIELD psfQfPerfMax, PSF_QF_PERF_MAX, float, best psfQfPerfect for this filter + +# these statistics are PSF-specific +FIELD Mstdev, MAG_STDEV, float, standard deviation of measurements, mags +FIELD Mmin, MAG_MIN, float, min accepted mag, mags +FIELD Mmax, MAG_MAX, float, max accepted mag, mags +FIELD Mchisq, MAG_CHI, float, chisq on average mag, value + +FIELD Ncode, NCODE, short, number of detections in band +FIELD Nused, NUSED, short, number of detections used in average +FIELD NusedKron, NUSED_KRON, short, number of detections used in average +FIELD NusedAp, NUSED_AP, short, number of detections used in average + +FIELD flags, FLAGS, uint32_t, photometry flags + +## *** this section is for stack values *** + +FIELD MpsfStk, MAG_PSF_STK, float, magnitude from stack (primary if available) +FIELD FpsfStk, FLUX_PSF_STK, float, flux from stack (primary if available) +FIELD dFpsfStk, FLUX_PSF_STK_ERR, float, mean flux psf error + +FIELD MkronStk, MAG_KRON_STK, float, magnitude from stack (primary if available) +FIELD FkronStk, FLUX_KRON_STK, float, flux from stack (primary if available) +FIELD dFkronStk, FLUX_KRON_STK_ERR, float, mean flux kron error + +FIELD MapStk, MAG_AP_STK, float, magnitude from stack (primary if available) +FIELD FapStk, FLUX_AP_STK, float, flux from stack (primary if available) +FIELD dFapStk, FLUX_AP_STK_ERR, float, mean flux ap error + +FIELD Nstack, NSTACK, short, number of measurements in band (gpc1 stack only) +FIELD NstackDet, NSTACK_DET, short, number of stack detections in band (gpc1 stack only) + +FIELD stackPrmryOff, STACK_PRIMARY_OFF, int, measure entry which is primary stack detection +FIELD stackBestOff, STACK_BEST_OFF, int, measure entry which is best stack detection + +## *** this section is for forced-warp mean values *** + +FIELD MpsfWrp, MAG_PSF_WRP, float, psf magnitude from stack (primary if available) +FIELD FpsfWrp, FLUX_PSF_WRP, float, psf flux from stack (primary if available) +FIELD dFpsfWrp, FLUX_PSF_WRP_ERR, float, mean flux psf error +FIELD sFpsfWrp, FLUX_PSF_WRP_STD, float, mean flux psf stdev + +FIELD MkronWrp, MAG_KRON_WRP, float, kron magnitude from stack (primary if available) +FIELD FkronWrp, FLUX_KRON_WRP, float, kron flux from stack (primary if available) +FIELD dFkronWrp, FLUX_KRON_WRP_ERR, float, mean flux kron error +FIELD sFkronWrp, FLUX_KRON_WRP_STD, float, mean flux kron stdev + +FIELD MapWrp, MAG_AP_WRP, float, aper magnitude from stack (primary if available) +FIELD FapWrp, FLUX_AP_WRP, float, aper flux from stack (primary if available) +FIELD dFapWrp, FLUX_AP_WRP_ERR, float, mean flux ap error +FIELD sFapWrp, FLUX_AP_WRP_STD, float, mean flux ap stdev + +FIELD NusedWrp, NUSED_WRP, short, number of detections used in average +FIELD NusedKronWrp, NUSED_KRON_WRP, short, number of detections used in average +FIELD NusedApWrp, NUSED_AP_WRP, short, number of detections used in average + +FIELD Nwarp, NWARP, short, number of measurements in band (gpc1 warp only) +FIELD NwarpGood, NWARP_GOOD, short, number of meas with psf_qf > 0.85 in band (gpc1 warp only) + +FIELD ubercalDist, UBERCAL_DIST, short, number of images from an ubercal-image + +# 31*float + uint32_t + uint64_t + 8*short = 152 bytes / object / photcode +# for 5G objects, expect ~6TB total (over 60 machines, this is 100G) Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/secfilt.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/secfilt.d (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/secfilt.d (revision 40477) @@ -5,13 +5,18 @@ DESCRIPTION DVO SecFilt : Secondary Filter Data +## 2017.11.22 : To clarify the parallel structures, I am explicitly renaming M,Map,Mkron to MpsfChp, MapChp, MkronChp + ## *** this section is for per-exposure mean values *** (unlabled values are implicitly PSF values) -FIELD M, MAG, float, average mag in this band, mags -FIELD dM, MAG_ERR, float, formal error on average mag, mags -FIELD Map, MAG_AP, float, average mag in this band, mags -FIELD dMap, MAG_AP_ERR, float, formal error on average mag, mags -FIELD sMap, MAG_AP_STDEV, float, standard deviation of ap mags, mags -FIELD Mkron, MAG_KRON, float, ave kron mag in this band, mags -FIELD dMkron, MAG_KRON_ERR, float, formal error on average kron mag, mags -FIELD sMkron, MAG_KRON_STDEV, float, standard deviation of kron mags, mags +FIELD MpsfChp, MAG, float, average mag in this band, mags +FIELD dMpsfChp, MAG_ERR, float, formal error on average mag, mags +FIELD sMpsfChp, MAG_STDEV, float, standard deviation of measurements, mags + +FIELD MapChp, MAG_AP, float, average mag in this band, mags +FIELD dMapChp, MAG_AP_ERR, float, formal error on average mag, mags +FIELD sMapChp, MAG_AP_STDEV, float, standard deviation of ap mags, mags + +FIELD MkronChp, MAG_KRON, float, ave kron mag in this band, mags +FIELD dMkronChp, MAG_KRON_ERR, float, formal error on average kron mag, mags +FIELD sMkronChp, MAG_KRON_STDEV, float, standard deviation of kron mags, mags # XXX I could add these fields to secfilt or calculate in dvopsps? @@ -20,5 +25,4 @@ # these statistics are PSF-specific -FIELD Mstdev, MAG_STDEV, float, standard deviation of measurements, mags FIELD Mmin, MAG_MIN, float, minimum accepted mag, mags FIELD Mmax, MAG_MAX, float, maximum accepted mag, mags Index: /branches/czw_branch/20170908/Ohana/src/libautocode/def/starpar-ps1-v6.d =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libautocode/def/starpar-ps1-v6.d (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libautocode/def/starpar-ps1-v6.d (revision 40477) @@ -0,0 +1,24 @@ +STRUCT StarPar_PS1_V6 +EXTNAME DVO_STARPAR_PS1_V6 +TYPE BINTABLE +SIZE 80 +DESCRIPTION DVO Table of Stellar Properties + +FIELD R, RA, double, ra +FIELD D, DEC, double, dec +FIELD galLat, GAL_LAT, float, galactic latitude +FIELD galLon, GAL_LON, float, galactic longitude +FIELD Ebv, E_BV, float, extinction +FIELD dEbv, E_BV_ERR, float, extinction error +FIELD DistMag, DISTANCE_MOD, float, mag +FIELD dDistMag, DISTANCE_MOD_ERR, float, mag +FIELD M_r, M_R_ABS, float, r-band abs magnitude +FIELD dM_r, M_R_ABS_ERR, float, r-band abs magnitude error +FIELD FeH, F_E_H, float, metallicity +FIELD dFeH, F_E_H_ERR, float, metallicity error +FIELD uRA, U_RA, float, model guess for proper motion +FIELD uDEC, U_DEC, float, model guess for proper motion +FIELD averef, AVE_REF, unsigned int, reference to average entry +FIELD objID, OBJ_ID, unsigned int, unique ID for object in table +FIELD catID, CAT_ID, unsigned int, unique ID for table in which object was first realized +FIELD dummy, DUMMY, unsigned int, dummy Index: /branches/czw_branch/20170908/Ohana/src/libdvo/Makefile =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/Makefile (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/Makefile (revision 40477) @@ -50,4 +50,5 @@ $(DESTINC)/ps1_v4_defs.h \ $(DESTINC)/ps1_v5_defs.h \ +$(DESTINC)/ps1_v6_defs.h \ $(DESTINC)/ps1_v5_ld_defs.h \ $(DESTINC)/ps1_ref_defs.h \ @@ -100,4 +101,5 @@ $(SRC)/dvo_convert_PS1_V4.$(ARCH).o \ $(SRC)/dvo_convert_PS1_V5.$(ARCH).o \ +$(SRC)/dvo_convert_PS1_V6.$(ARCH).o \ $(SRC)/dvo_convert_PS1_V5_LOAD.$(ARCH).o \ $(SRC)/dvo_convert_PS1_REF.$(ARCH).o \ Index: /branches/czw_branch/20170908/Ohana/src/libdvo/doc/notes.txt =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/doc/notes.txt (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/doc/notes.txt (revision 40477) @@ -16,5 +16,5 @@ uppercase, eg: AveragePanstarrs_DEV_0, AveragePanstarrs_PS1_2). -4) create a DVOTableFormat entry for the new format +4) create a DVOCatFormat entry for the new format (libdvo/include/dvo.h). the naming convention is DVO_FORMAT_FOO. @@ -37,5 +37,7 @@ make is easy to identify the new changes needed. -10) add the new format to the section at the end of dvo_image.c +10a) if the Image structure must be changed, this is done in libdvo/include/libdvo_astro.h, not autocode +10b) add the new format to the section at the end of dvo_image.c +10c) update gfits_table_mkheader_Image in dvo_image.c to match modifications to Image (if any) 11) add the new format to the dvo_image_raw.c Index: /branches/czw_branch/20170908/Ohana/src/libdvo/include/dvo.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/include/dvo.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/include/dvo.h (revision 40477) @@ -33,4 +33,5 @@ DVO_FORMAT_PS1_V4, DVO_FORMAT_PS1_V5, + DVO_FORMAT_PS1_V6, DVO_FORMAT_PS1_V5_LOAD, } DVOCatFormat; @@ -220,31 +221,32 @@ /* Secfilt.flags values -- these values are 32 bit (as of PS1_V1) */ typedef enum { - ID_SECF_STAR_FEW = 0x00000001, // used within relphot: skip star - ID_SECF_STAR_POOR = 0x00000002, // used within relphot: skip star - ID_SECF_USE_SYNTH = 0x00000004, // synthetic photometry used in average measurement - ID_SECF_USE_UBERCAL = 0x00000008, // ubercal photometry used in average measurement - ID_SECF_HAS_PS1 = 0x00000010, // PS1 photometry used in average measurement - ID_SECF_HAS_PS1_STACK = 0x00000020, // PS1 stack photometry exists - ID_SECF_HAS_TYCHO = 0x00000040, // Tycho photometry used for synth mags - ID_SECF_FIX_SYNTH = 0x00000080, // synth mags repaired with zpt map - ID_SECF_RANK_0 = 0x00000100, // average magnitude uses rank 0 values - ID_SECF_RANK_1 = 0x00000200, // average magnitude uses rank 1 values - ID_SECF_RANK_2 = 0x00000400, // average magnitude uses rank 2 values - ID_SECF_RANK_3 = 0x00000800, // average magnitude uses rank 3 values - ID_SECF_RANK_4 = 0x00001000, // average magnitude uses rank 4 values - ID_SECF_OBJ_EXT_PSPS = 0x00002000, // In PSPS ID_SECF_OBJ_EXT is moved here so it fits within 16 bits - ID_SECF_STACK_PRIMARY = 0x00004000, // PS1 stack photometry comes from primary skycell - ID_SECF_STACK_BESTDET = 0x00008000, // PS1 stack best measurement is a detection (not forced) - ID_SECF_STACK_PRIMDET = 0x00010000, // PS1 stack primary measurement is a detection (not forced) - - ID_SECF_HAS_SDSS = 0x00100000, // this photcode has SDSS photometry - ID_SECF_HAS_HSC = 0x00200000, // this photcode has HSC photometry - ID_SECF_HAS_CFH = 0x00400000, // this photcode has CFH photometry (mostly Megacam) - ID_SECF_HAS_DES = 0x00800000, // this photcode has DES photometry - - ID_SECF_OBJ_EXT = 0x01000000, // extended in this band - - ID_SECF_CHIP_FLAGS = 0x01003f1f, // all chip-related bits (used to reset the correct bits only) - ID_SECF_STACK_FLAGS = 0x00004020, // all stack-related bits ( + ID_SECF_STAR_FEW = 0x00000001, // used within relphot: skip star + ID_SECF_STAR_POOR = 0x00000002, // used within relphot: skip star + ID_SECF_USE_SYNTH = 0x00000004, // synthetic photometry used in average measurement + ID_SECF_USE_UBERCAL = 0x00000008, // ubercal photometry used in average measurement + ID_SECF_HAS_PS1 = 0x00000010, // PS1 photometry used in average measurement + ID_SECF_HAS_PS1_STACK = 0x00000020, // PS1 stack photometry exists + ID_SECF_HAS_TYCHO = 0x00000040, // Tycho photometry used for synth mags + ID_SECF_FIX_SYNTH = 0x00000080, // synth mags repaired with zpt map + ID_SECF_RANK_0 = 0x00000100, // average magnitude uses rank 0 values + ID_SECF_RANK_1 = 0x00000200, // average magnitude uses rank 1 values + ID_SECF_RANK_2 = 0x00000400, // average magnitude uses rank 2 values + ID_SECF_RANK_3 = 0x00000800, // average magnitude uses rank 3 values + ID_SECF_RANK_4 = 0x00001000, // average magnitude uses rank 4 values + ID_SECF_OBJ_EXT_PSPS = 0x00002000, // In PSPS ID_SECF_OBJ_EXT is moved here so it fits within 16 bits + ID_SECF_STACK_PRIMARY = 0x00004000, // PS1 stack photometry includes a primary skycell + ID_SECF_STACK_BESTDET = 0x00008000, // PS1 stack best measurement is a detection (not forced) + ID_SECF_STACK_PRIMDET = 0x00010000, // PS1 stack primary measurement is a detection (not forced) + ID_SECF_STACK_PRIMARY_MULTIPLE = 0x00020000, // PS1 stack object has multiple primary measurements + + ID_SECF_HAS_SDSS = 0x00100000, // this photcode has SDSS photometry + ID_SECF_HAS_HSC = 0x00200000, // this photcode has HSC photometry + ID_SECF_HAS_CFH = 0x00400000, // this photcode has CFH photometry (mostly Megacam) + ID_SECF_HAS_DES = 0x00800000, // this photcode has DES photometry + + ID_SECF_OBJ_EXT = 0x01000000, // extended in this band + + ID_SECF_CHIP_FLAGS = 0x01003f1f, // all chip-related bits (used to reset the correct bits only) + ID_SECF_STACK_FLAGS = 0x0003c020, // all stack-related bits ( } DVOSecfiltFlags; @@ -501,6 +503,8 @@ double R; double D; - float M; - float Mcal; + float M; // change to Mpsf eventually to disambiguate + float Mkron; + float McalPSF; + float McalAPER; float Mflat; float dM; @@ -1073,4 +1077,5 @@ # include "ps1_v4_defs.h" # include "ps1_v5_defs.h" +# include "ps1_v6_defs.h" # include "ps1_v5_ld_defs.h" # include "ps1_ref_defs.h" @@ -1267,19 +1272,39 @@ # undef LENSFIELD -# define GALPHOT_FIELD(NAME, VALUE) float GalphotValue_##NAME (PhotCode *code, dvoMagClassType class, GalPhot *galphot, int Ngalphot) - -GALPHOT_FIELD(GAL_MAG, mag); -GALPHOT_FIELD(GAL_MAG_ERR, magErr); -GALPHOT_FIELD(GAL_MAJ, majorAxis); -GALPHOT_FIELD(GAL_MAJ_ERR, majorAxisErr); -GALPHOT_FIELD(GAL_MIN, minorAxis); -GALPHOT_FIELD(GAL_MIN_ERR, minorAxisErr); -GALPHOT_FIELD(GAL_THETA, theta); -GALPHOT_FIELD(GAL_THETA_ERR, thetaErr); -GALPHOT_FIELD(GAL_INDEX, index); -GALPHOT_FIELD(GAL_CHISQ, chisq); -GALPHOT_FIELD(GAL_NPIX, Npix); -GALPHOT_FIELD(GAL_FLAGS, flags); -GALPHOT_FIELD(GAL_TYPE, modelType); +# define GALPHOT_FIELD(NAME, VALUE, TYPE, DEFAULT) TYPE GalphotValue_##NAME (PhotCode *code, dvoMagClassType class, GalPhot *galphot, int Ngalphot); + +//GALPHOT_FIELD(GAL_MAG, mag); +//GALPHOT_FIELD(GAL_MAG_ERR, magErr); +//GALPHOT_FIELD(GAL_MAJ, majorAxis); +//GALPHOT_FIELD(GAL_MAJ_ERR, majorAxisErr); +//GALPHOT_FIELD(GAL_MIN, minorAxis); +//GALPHOT_FIELD(GAL_MIN_ERR, minorAxisErr); +//GALPHOT_FIELD(GAL_THETA, theta); +//GALPHOT_FIELD(GAL_THETA_ERR, thetaErr); +//GALPHOT_FIELD(GAL_INDEX, index); +//GALPHOT_FIELD(GAL_CHISQ, chisq); +//GALPHOT_FIELD(GAL_NPIX, Npix); +//GALPHOT_FIELD(GAL_FLAGS, flags); +//GALPHOT_FIELD(GAL_TYPE, modelType); + +GALPHOT_FIELD(GAL_MAG, mag, float, NAN) +GALPHOT_FIELD(GAL_MAG_ERR, magErr, float, NAN) +GALPHOT_FIELD(GAL_MAJ, majorAxis, float, NAN) +GALPHOT_FIELD(GAL_MAJ_ERR, majorAxisErr, float, NAN) +GALPHOT_FIELD(GAL_MIN, minorAxis, float, NAN) +GALPHOT_FIELD(GAL_MIN_ERR, minorAxisErr, float, NAN) +GALPHOT_FIELD(GAL_THETA, theta, float, NAN) +GALPHOT_FIELD(GAL_THETA_ERR, thetaErr, float, NAN) +GALPHOT_FIELD(GAL_INDEX, index, float, NAN) +GALPHOT_FIELD(GAL_CHISQ, chisq, float, NAN) +GALPHOT_FIELD(GAL_NPIX, Npix, float, NAN) + +GALPHOT_FIELD(GAL_TYPE, modelType, short, 0) + +GALPHOT_FIELD(GAL_FLAGS, flags, unsigned int, 0) +GALPHOT_FIELD(GAL_OBJ_ID, objID, unsigned int, 0) +GALPHOT_FIELD(GAL_CAT_ID, catID, unsigned int, 0) +GALPHOT_FIELD(GAL_DET_ID, detID, unsigned int, 0) +GALPHOT_FIELD(GAL_IMAGE_ID, imageID, unsigned int, 0) # undef GALPHOT_FIELD Index: /branches/czw_branch/20170908/Ohana/src/libdvo/include/dvodb.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/include/dvodb.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/include/dvodb.h (revision 40477) @@ -3,4 +3,11 @@ # define MEASURE_HAS_XCCD 1 + +// Some values used by code moved to libdvo from opihi. +enum {OPIHI_NOTYPE, OPIHI_FLT, OPIHI_INT}; +# define opihi_flt double +// # define opihi_int int64_t +# define opihi_int long long int +# define OPIHI_INT_FMT "%lld" typedef enum { @@ -120,4 +127,9 @@ MAG_OPTION_GAL_FLAGS, MAG_OPTION_GAL_TYPE, + + MAG_OPTION_GAL_OBJ_ID, + MAG_OPTION_GAL_CAT_ID, + MAG_OPTION_GAL_DET_ID, + MAG_OPTION_GAL_IMAGE_ID, } dvoMagOptionType; @@ -236,5 +248,6 @@ MEAS_EXTERN_ID, MEAS_EXPNAME_AS_INT, - MEAS_MCAL_OFFSET, // make this a dvoMagOption? + MEAS_MCAL_OFFSET_PSF, // make this a dvoMagOption? + MEAS_MCAL_OFFSET_APER, // make this a dvoMagOption? MEAS_FLAT, MEAS_CENTER_OFFSET, @@ -369,5 +382,6 @@ IMAGE_XM, IMAGE_AIRMASS, - IMAGE_MCAL, + IMAGE_MCAL_PSF, + IMAGE_MCAL_APER, IMAGE_dMCAL, IMAGE_PHOTCODE, @@ -471,15 +485,15 @@ char type; int field; - // opihi_flt FltValue; - // opihi_int IntValue; - double FltValue; - int IntValue; + opihi_flt FltValue; + opihi_int IntValue; + // double FltValue; + // int IntValue; } dbStack; typedef struct { - // opihi_flt Flt; - // opihi_int Int; - double Flt; - int Int; + opihi_flt Flt; + opihi_int Int; + // double Flt; + // int Int; } dbValue; @@ -562,9 +576,4 @@ int dbExtractImagesReset (void); -// Some values used by code moved to libdvo from opihi. -enum {OPIHI_NOTYPE, OPIHI_FLT, OPIHI_INT}; -#define opihi_flt double -#define opihi_int int - #include "get_graphdata.h" Index: /branches/czw_branch/20170908/Ohana/src/libdvo/include/libdvo_astro.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/include/libdvo_astro.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/include/libdvo_astro.h (revision 40477) @@ -100,7 +100,8 @@ float apmifit; // aperture correction (mag) float dapmifit; // apmifit error (mag) - float Mcal; // calibration mag (mag) + float McalPSF; // calibration mag (mag) + float McalAPER; // calibration mag (mag) float dMcal; // error on Mcal (mag) - short Xm; // image chisq (10*log(value)) + float McalChiSq; // image chisq (10*log(value)) short photcode; // identifier for CCD, float exptime; // exposure time (seconds) Index: /branches/czw_branch/20170908/Ohana/src/libdvo/include/ps1_v6_defs.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/include/ps1_v6_defs.h (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/include/ps1_v6_defs.h (revision 40477) @@ -0,0 +1,26 @@ +Image *Image_PS1_V6_ToInternal (Image_PS1_V6 *in, off_t Nvalues, off_t Nalloc); +Image_PS1_V6 *ImageInternalTo_PS1_V6 (Image *in, off_t Nvalues); + +Average *Average_PS1_V6_ToInternal (Average_PS1_V6 *in, off_t Nvalues, SecFilt **primary); +Average_PS1_V6 *AverageInternalTo_PS1_V6 (Average *in, off_t Nvalues, SecFilt *primary); + +Measure *Measure_PS1_V6_ToInternal (Average *ave, Measure_PS1_V6 *in, off_t Nvalues); +Measure_PS1_V6 *MeasureInternalTo_PS1_V6 (Average *ave, Measure *in, off_t Nvalues); + +SecFilt *SecFilt_PS1_V6_ToInternal (SecFilt_PS1_V6 *in, off_t Nvalues); +SecFilt_PS1_V6 *SecFiltInternalTo_PS1_V6 (SecFilt *in, off_t Nvalues); + +Lensobj *Lensobj_PS1_V6_ToInternal (Lensobj_PS1_V6 *in, off_t Nvalues); +Lensobj_PS1_V6 *LensobjInternalTo_PS1_V6 (Lensobj *in, off_t Nvalues); + +Lensing *Lensing_PS1_V6_ToInternal (Lensing_PS1_V6 *in, off_t Nvalues); +Lensing_PS1_V6 *LensingInternalTo_PS1_V6 (Lensing *in, off_t Nvalues); + +StarPar *StarPar_PS1_V6_ToInternal (StarPar_PS1_V6 *in, off_t Nvalues); +StarPar_PS1_V6 *StarParInternalTo_PS1_V6 (StarPar *in, off_t Nvalues); + +GalPhot *GalPhot_PS1_V6_ToInternal (GalPhot_PS1_V6 *in, off_t Nvalues); +GalPhot_PS1_V6 *GalPhotInternalTo_PS1_V6 (GalPhot *in, off_t Nvalues); + +PhotCode *PhotCode_PS1_V6_To_Internal (PhotCode_PS1_V6 *in, off_t Nvalues); +PhotCode_PS1_V6 *PhotCode_Internal_To_PS1_V6 (PhotCode *in, off_t Nvalues); Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/HostTable.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/HostTable.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/HostTable.c (revision 40477) @@ -11,4 +11,5 @@ host->stdio[HOST_STDERR] = -1; host->pid = 0; + host->status = 1024; return; } Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/ImageMetadata.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/ImageMetadata.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/ImageMetadata.c (revision 40477) @@ -64,5 +64,5 @@ GET_COLUMN (crval2, "CRVAL2", double); GET_COLUMN (theta, "THETA", float); - GET_COLUMN (Mcal, "MCAL", float); + GET_COLUMN (McalPSF, "MCAL", float); GET_COLUMN (secz, "SECZ", float); GET_COLUMN (Xcenter, "X_CENTER", float); @@ -77,5 +77,5 @@ image[i].crval2 = crval2[i] ; image[i].theta = theta[i] ; - image[i].Mcal = Mcal[i] ; + image[i].Mcal = McalPSF[i] ; image[i].secz = secz[i] ; image[i].Xcenter = Xcenter[i] ; @@ -90,5 +90,5 @@ free (crval2); free (theta); - free (Mcal); + free (McalPSF); free (secz); free (Xcenter); @@ -166,5 +166,5 @@ theta[i] = DEG_RAD*atan2(mosaic->pc1_2, mosaic->pc1_1); - Mcal[i] = image[i].Mcal; + Mcal[i] = image[i].McalPSF; secz[i] = image[i].secz; Xcenter[i] = 0.5*image[i].NX; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/LoadPhotcodesFITS.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/LoadPhotcodesFITS.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/LoadPhotcodesFITS.c (revision 40477) @@ -64,4 +64,5 @@ CONVERT_FORMAT("DVO_PHOTCODE_PS1_V4", PS1_V4); CONVERT_FORMAT("DVO_PHOTCODE_PS1_V5", PS1_V5); + CONVERT_FORMAT("DVO_PHOTCODE_PS1_V6", PS1_V6); CONVERT_FORMAT("DVO_PHOTCODE_PS1_V5_LOAD", PS1_V5_LOAD); Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/SavePhotcodesFITS.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/SavePhotcodesFITS.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/SavePhotcodesFITS.c (revision 40477) @@ -30,9 +30,9 @@ // for the moment, we simply support the latest photcode format for output // XXX update this as needed as new formats are defined - PhotCode_PS1_V5 *photcode_output = PhotCode_Internal_To_PS1_V5 (table[0].code, table[0].Ncode); + PhotCode_PS1_V6 *photcode_output = PhotCode_Internal_To_PS1_V6 (table[0].code, table[0].Ncode); /* convert FITS format data to internal format (byteswaps & EXTNAME) */ if (!gfits_db_create (&db)) return (FALSE); - if (!gfits_table_set_PhotCode_PS1_V5 (&db.ftable, photcode_output, table[0].Ncode, TRUE)) return (FALSE); + if (!gfits_table_set_PhotCode_PS1_V6 (&db.ftable, photcode_output, table[0].Ncode, TRUE)) return (FALSE); if (!gfits_db_save (&db)) return (FALSE); if (!gfits_db_close (&db)) return (FALSE); Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbCheckStack.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbCheckStack.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbCheckStack.c (revision 40477) @@ -43,5 +43,5 @@ * an int unless proven otherwise **/ stack[i].FltValue = strtod (stack[i].name, &c1); - stack[i].IntValue = strtol (stack[i].name, &c2, 0); + stack[i].IntValue = strtoll (stack[i].name, &c2, 0); if (c2 == stack[i].name + strlen (stack[i].name)) { stack[i].type |= DB_STACK_INT; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbExtractAverages.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbExtractAverages.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbExtractAverages.c (revision 40477) @@ -466,17 +466,22 @@ break; - case MAG_OPTION_GAL_MAG: { value.Flt = GalphotValue_GAL_MAG (field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_MAG_ERR: { value.Flt = GalphotValue_GAL_MAG_ERR (field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_MAJ: { value.Flt = GalphotValue_GAL_MAJ (field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_MAJ_ERR: { value.Flt = GalphotValue_GAL_MAJ_ERR (field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_MIN: { value.Flt = GalphotValue_GAL_MIN (field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_MIN_ERR: { value.Flt = GalphotValue_GAL_MIN_ERR (field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_THETA: { value.Flt = GalphotValue_GAL_THETA (field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_THETA_ERR: { value.Flt = GalphotValue_GAL_THETA_ERR(field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_INDEX: { value.Flt = GalphotValue_GAL_INDEX (field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_CHISQ: { value.Flt = GalphotValue_GAL_CHISQ (field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_NPIX: { value.Flt = GalphotValue_GAL_NPIX (field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_TYPE: { value.Flt = GalphotValue_GAL_TYPE (field->photcode, field->magClass, galphot, average->Ngalphot); break; } - case MAG_OPTION_GAL_FLAGS: { value.Flt = GalphotValue_GAL_FLAGS (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_MAG: { value.Flt = GalphotValue_GAL_MAG (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_MAG_ERR: { value.Flt = GalphotValue_GAL_MAG_ERR (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_MAJ: { value.Flt = GalphotValue_GAL_MAJ (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_MAJ_ERR: { value.Flt = GalphotValue_GAL_MAJ_ERR (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_MIN: { value.Flt = GalphotValue_GAL_MIN (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_MIN_ERR: { value.Flt = GalphotValue_GAL_MIN_ERR (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_THETA: { value.Flt = GalphotValue_GAL_THETA (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_THETA_ERR:{ value.Flt = GalphotValue_GAL_THETA_ERR(field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_INDEX: { value.Flt = GalphotValue_GAL_INDEX (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_CHISQ: { value.Flt = GalphotValue_GAL_CHISQ (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_NPIX: { value.Flt = GalphotValue_GAL_NPIX (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_TYPE: { value.Flt = GalphotValue_GAL_TYPE (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_FLAGS: { value.Flt = GalphotValue_GAL_FLAGS (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + + case MAG_OPTION_GAL_OBJ_ID: { value.Flt = GalphotValue_GAL_OBJ_ID (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_CAT_ID: { value.Flt = GalphotValue_GAL_CAT_ID (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_DET_ID: { value.Flt = GalphotValue_GAL_DET_ID (field->photcode, field->magClass, galphot, average->Ngalphot); break; } + case MAG_OPTION_GAL_IMAGE_ID: { value.Flt = GalphotValue_GAL_IMAGE_ID (field->photcode, field->magClass, galphot, average->Ngalphot); break; } default: Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbExtractImages.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbExtractImages.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbExtractImages.c (revision 40477) @@ -219,6 +219,9 @@ break; - case IMAGE_MCAL: - value.Flt = image[N].Mcal; + case IMAGE_MCAL_PSF: + value.Flt = image[N].McalPSF; + break; + case IMAGE_MCAL_APER: + value.Flt = image[N].McalAPER; break; case IMAGE_dMCAL: @@ -226,5 +229,5 @@ break; case IMAGE_XM: - value.Flt = pow(10.0, 0.01*image[N].Xm); + value.Flt = image[N].McalChiSq; break; case IMAGE_PHOTCODE: @@ -374,5 +377,5 @@ break; case IMAGE_Y_ERR_SYS: - value.Flt = image[N].dXpixSys; + value.Flt = image[N].dYpixSys; break; case IMAGE_MAG_ERR_SYS: Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbExtractMeasures.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 40477) @@ -316,4 +316,8 @@ case MAG_OPTION_GAL_FLAGS: case MAG_OPTION_GAL_TYPE: + case MAG_OPTION_GAL_OBJ_ID: + case MAG_OPTION_GAL_CAT_ID: + case MAG_OPTION_GAL_DET_ID: + case MAG_OPTION_GAL_IMAGE_ID: case MAG_OPTION_NONE: @@ -696,5 +700,5 @@ break; case MEAS_PSF_QF_PERFECT: /* OK */ - value.Flt = NAN; + value.Flt = measure[0].psfQFperf; break; case MEAS_PSF_CHISQ: /* OK */ @@ -740,5 +744,6 @@ break; - case MEAS_MCAL_OFFSET: { value.Flt = measure[0].Mcal; break; } + case MEAS_MCAL_OFFSET_PSF: { value.Flt = measure[0].McalPSF; break; } + case MEAS_MCAL_OFFSET_APER: { value.Flt = measure[0].McalAPER; break; } case MEAS_FLAT: { value.Flt = measure[0].Mflat; break; } Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbFields.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbFields.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dbFields.c (revision 40477) @@ -181,4 +181,9 @@ if (magOption == MAG_OPTION_GAL_FLAGS ) return TRUE; if (magOption == MAG_OPTION_GAL_TYPE ) return TRUE; + + if (magOption == MAG_OPTION_GAL_OBJ_ID ) return TRUE; + if (magOption == MAG_OPTION_GAL_CAT_ID ) return TRUE; + if (magOption == MAG_OPTION_GAL_DET_ID ) return TRUE; + if (magOption == MAG_OPTION_GAL_IMAGE_ID ) return TRUE; return FALSE; } @@ -609,5 +614,7 @@ if (!strcasecmp (fieldName, "SKY")) ESCAPE (MEAS_SKY, OPIHI_FLT); if (!strcasecmp (fieldName, "SKY_ERR")) ESCAPE (MEAS_dSKY, OPIHI_FLT); - if (!strcasecmp (fieldName, "MCAL_OFFSET")) ESCAPE (MEAS_MCAL_OFFSET, OPIHI_FLT); + if (!strcasecmp (fieldName, "MCAL_OFFSET")) ESCAPE (MEAS_MCAL_OFFSET_PSF, OPIHI_FLT); + if (!strcasecmp (fieldName, "MCAL_OFFSET_PSF")) ESCAPE (MEAS_MCAL_OFFSET_PSF, OPIHI_FLT); + if (!strcasecmp (fieldName, "MCAL_OFFSET_APER")) ESCAPE (MEAS_MCAL_OFFSET_APER, OPIHI_FLT); if (!strcasecmp (fieldName, "FLAT")) ESCAPE (MEAS_FLAT, OPIHI_FLT); if (!strcasecmp (fieldName, "CENTER_OFFSET")) ESCAPE (MEAS_CENTER_OFFSET, OPIHI_FLT); @@ -845,11 +852,14 @@ if (!strcasecmp (fieldName, "dapresid" )) ESCAPE (IMAGE_DAPRESID, OPIHI_FLT); - if (!strcasecmp (fieldName, "Mcal" )) ESCAPE (IMAGE_MCAL, OPIHI_FLT); - if (!strcasecmp (fieldName, "dMcal" )) ESCAPE (IMAGE_dMCAL, OPIHI_FLT); - if (!strcasecmp (fieldName, "Xm" )) ESCAPE (IMAGE_XM, OPIHI_FLT); - if (!strcasecmp (fieldName, "photcode" )) ESCAPE (IMAGE_PHOTCODE, OPIHI_INT); - if (!strcasecmp (fieldName, "exptime" )) ESCAPE (IMAGE_EXPTIME, OPIHI_FLT); - if (!strcasecmp (fieldName, "expname" )) ESCAPE (IMAGE_EXPNAME_AS_INT, OPIHI_INT); - if (!strcasecmp (fieldName, "sidtime" )) ESCAPE (IMAGE_SIDTIME, OPIHI_FLT); + if (!strcasecmp (fieldName, "Mcal" )) ESCAPE (IMAGE_MCAL_PSF, OPIHI_FLT); + if (!strcasecmp (fieldName, "McalPSF" )) ESCAPE (IMAGE_MCAL_PSF, OPIHI_FLT); + if (!strcasecmp (fieldName, "McalAPER" )) ESCAPE (IMAGE_MCAL_APER, OPIHI_FLT); + if (!strcasecmp (fieldName, "McalAPERTURE")) ESCAPE (IMAGE_MCAL_APER, OPIHI_FLT); + if (!strcasecmp (fieldName, "dMcal" )) ESCAPE (IMAGE_dMCAL, OPIHI_FLT); + if (!strcasecmp (fieldName, "Xm" )) ESCAPE (IMAGE_XM, OPIHI_FLT); + if (!strcasecmp (fieldName, "photcode" )) ESCAPE (IMAGE_PHOTCODE, OPIHI_INT); + if (!strcasecmp (fieldName, "exptime" )) ESCAPE (IMAGE_EXPTIME, OPIHI_FLT); + if (!strcasecmp (fieldName, "expname" )) ESCAPE (IMAGE_EXPNAME_AS_INT, OPIHI_INT); + if (!strcasecmp (fieldName, "sidtime" )) ESCAPE (IMAGE_SIDTIME, OPIHI_FLT); if (!strcasecmp (fieldName, "latitude" )) ESCAPE (IMAGE_LATITUDE, OPIHI_FLT); Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_catalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_catalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_catalog.c (revision 40477) @@ -57,4 +57,5 @@ if (!strcasecmp (catformat, "PS1_V4")) return (DVO_FORMAT_PS1_V4); if (!strcasecmp (catformat, "PS1_V5")) return (DVO_FORMAT_PS1_V5); + if (!strcasecmp (catformat, "PS1_V6")) return (DVO_FORMAT_PS1_V6); if (!strcasecmp (catformat, "PS1_V5_LOAD")) return (DVO_FORMAT_PS1_V5_LOAD); if (!strcasecmp (catformat, "PS1_REF")) return (DVO_FORMAT_PS1_REF); @@ -234,17 +235,17 @@ if (mode & SECFILT_RESET_CHIP) { - secfilt->M = NAN; - secfilt->dM = NAN; - secfilt->Map = NAN; - secfilt->dMap = NAN; - secfilt->sMap = NAN; - secfilt->Mkron = NAN; - secfilt->dMkron = NAN; - secfilt->sMkron = NAN; + secfilt->MpsfChp = NAN; + secfilt->dMpsfChp = NAN; + secfilt->sMpsfChp = NAN; + secfilt->MapChp = NAN; + secfilt->dMapChp = NAN; + secfilt->sMapChp = NAN; + secfilt->MkronChp = NAN; + secfilt->dMkronChp = NAN; + secfilt->sMkronChp = NAN; secfilt->psfQfMax = NAN; secfilt->psfQfPerfMax = NAN; - secfilt->Mstdev = NAN; secfilt->Mmin = NAN; secfilt->Mmax = NAN; @@ -316,5 +317,6 @@ measure->Mkron = NAN; measure->dMkron = NAN; - measure->Mcal = NAN; + measure->McalPSF = NAN; + measure->McalAPER = NAN; measure->dMcal = NAN; measure->dt = NAN; @@ -343,5 +345,4 @@ measure->Mflat = 0.0; - measure->dummy2 = 0; measure->Sky = NAN; @@ -394,5 +395,6 @@ measure->D = NAN; measure->M = NAN; - measure->Mcal = NAN; + measure->McalPSF = NAN; + measure->McalAPER = NAN; measure->dM = NAN; @@ -922,12 +924,12 @@ for (in = out = i = 0; i < catalog[0].Naverage; i++) { for (j = 0; j < catalog[0].Nsecfilt; j++, in++, out++) { - outsec[out].M = insec[in].M; - outsec[out].dM = insec[in].dM; - outsec[out].Mchisq = insec[in].Mchisq; + outsec[out].MpsfChp = insec[in].MpsfChp; + outsec[out].dMpsfChp = insec[in].dMpsfChp; + outsec[out].Mchisq = insec[in].Mchisq; } for (j = 0; j < Nextra; j++, out++) { - outsec[out].M = NAN; - outsec[out].dM = NAN; - outsec[out].Mchisq = NAN; + outsec[out].MpsfChp = NAN; + outsec[out].dMpsfChp = NAN; + outsec[out].Mchisq = NAN; } } Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_catalog_raw.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 40477) @@ -91,4 +91,5 @@ FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); FORMAT_CASE (PS1_REF, PS1_REF); @@ -288,4 +289,5 @@ if (catalog[0].catformat == DVO_FORMAT_PS1_V4) gfits_modify (&catalog[0].header, "FORMAT", "%s", 1, "PS1_V4"); if (catalog[0].catformat == DVO_FORMAT_PS1_V5) gfits_modify (&catalog[0].header, "FORMAT", "%s", 1, "PS1_V5"); + if (catalog[0].catformat == DVO_FORMAT_PS1_V6) gfits_modify (&catalog[0].header, "FORMAT", "%s", 1, "PS1_V6"); if (catalog[0].catformat == DVO_FORMAT_PS1_V5_LOAD) gfits_modify (&catalog[0].header, "FORMAT", "%s", 1, "PS1_V5_LOAD"); if (catalog[0].catformat == DVO_FORMAT_PS1_REF) gfits_modify (&catalog[0].header, "FORMAT", "%s", 1, "PS1_REF"); @@ -394,4 +396,5 @@ FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); FORMAT_CASE (PS1_REF, PS1_REF); @@ -448,4 +451,5 @@ FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); FORMAT_CASE (PS1_REF, PS1_REF); @@ -505,4 +509,5 @@ FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); FORMAT_CASE (PS1_REF, PS1_REF); @@ -559,4 +564,5 @@ FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); FORMAT_CASE (PS1_REF, PS1_REF); @@ -617,4 +623,5 @@ FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); FORMAT_CASE (PS1_REF, PS1_REF); @@ -671,4 +678,5 @@ FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); FORMAT_CASE (PS1_REF, PS1_REF); Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_catalog_split.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 40477) @@ -464,7 +464,7 @@ catalog[0].Nmissing_disk = Nmissing; catalog[0].Nsecfilt_disk = Naverage * Nsecfilt; - catalog[0].Nlensing_disk = Nlensing; - catalog[0].Nlensobj_disk = Nlensobj; - catalog[0].Nstarpar_disk = Nstarpar; + catalog[0].Nlensing_disk = Nlensing; + catalog[0].Nlensobj_disk = Nlensobj; + catalog[0].Nstarpar_disk = Nstarpar; catalog[0].Ngalphot_disk = Ngalphot; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert.c (revision 40477) @@ -42,4 +42,5 @@ CONVERT_FORMAT ("DVO_AVERAGE_PS1_V4", PS1_V4); CONVERT_FORMAT ("DVO_AVERAGE_PS1_V5", PS1_V5); + CONVERT_FORMAT ("DVO_AVERAGE_PS1_V6", PS1_V6); CONVERT_FORMAT ("DVO_AVERAGE_PS1_V5_LOAD", PS1_V5_LOAD); CONVERT_FORMAT ("DVO_AVERAGE_PS1_REF", PS1_REF); @@ -133,4 +134,5 @@ CONVERT_FORMAT ("DVO_AVERAGE_PS1_V4", PS1_V4, PS1_V4); CONVERT_FORMAT ("DVO_AVERAGE_PS1_V5", PS1_V5, PS1_V5); + CONVERT_FORMAT ("DVO_AVERAGE_PS1_V6", PS1_V6, PS1_V6); CONVERT_FORMAT ("DVO_AVERAGE_PS1_V5_LOAD", PS1_V5_LOAD, PS1_V5_LOAD); CONVERT_FORMAT ("DVO_AVERAGE_PS1_REF", PS1_REF, PS1_REF); @@ -171,4 +173,5 @@ FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); FORMAT_CASE (PS1_REF, PS1_REF); @@ -270,4 +273,5 @@ CONVERT_FORMAT ("DVO_MEASURE_PS1_V4", PS1_V4, PS1_V4, FALSE); CONVERT_FORMAT ("DVO_MEASURE_PS1_V5", PS1_V5, PS1_V5, TRUE); + CONVERT_FORMAT ("DVO_MEASURE_PS1_V6", PS1_V6, PS1_V6, TRUE); CONVERT_FORMAT ("DVO_MEASURE_PS1_V5_LOAD", PS1_V5_LOAD, PS1_V5_LOAD, TRUE); CONVERT_FORMAT ("DVO_MEASURE_PS1_REF", PS1_REF, PS1_REF, FALSE); @@ -310,4 +314,5 @@ FORMAT_CASE (PS1_V4, PS1_V4, FALSE); FORMAT_CASE (PS1_V5, PS1_V5, TRUE); + FORMAT_CASE (PS1_V6, PS1_V6, TRUE); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD, TRUE); FORMAT_CASE (PS1_REF, PS1_REF, FALSE); @@ -391,4 +396,5 @@ SKIPPING_FORMAT ("DVO_MISSING_PS1_V4", PS1_V4, PS1_V4); SKIPPING_FORMAT ("DVO_MISSING_PS1_V5", PS1_V5, PS1_V5); + SKIPPING_FORMAT ("DVO_MISSING_PS1_V6", PS1_V6, PS1_V6); SKIPPING_FORMAT ("DVO_MISSING_PS1_V5_LOAD", PS1_V5_LOAD, PS1_V5_LOAD); # undef CONVERT_FORMAT @@ -466,4 +472,5 @@ CONVERT_FORMAT ("DVO_SECFILT_PS1_V4", PS1_V4, PS1_V4); CONVERT_FORMAT ("DVO_SECFILT_PS1_V5", PS1_V5, PS1_V5); + CONVERT_FORMAT ("DVO_SECFILT_PS1_V6", PS1_V6, PS1_V6); CONVERT_FORMAT ("DVO_SECFILT_PS1_V5_LOAD", PS1_V5_LOAD, PS1_V5_LOAD); CONVERT_FORMAT ("DVO_SECFILT_PS1_REF", PS1_REF, PS1_REF); @@ -504,4 +511,5 @@ FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); FORMAT_CASE (PS1_REF, PS1_REF); @@ -612,4 +620,5 @@ SKIPPING_FORMAT ("DVO_LENSING_PS1_V4", PS1_V4, PS1_V4); CONVERT_FORMAT ("DVO_LENSING_PS1_V5", PS1_V5, PS1_V5_R3); + CONVERT_FORMAT ("DVO_LENSING_PS1_V6", PS1_V6, PS1_V6); CONVERT_FORMAT ("DVO_LENSING_PS1_V5_LOAD", PS1_V5_LOAD, PS1_V5_LOAD); SKIPPING_FORMAT ("DVO_LENSING_PS1_SIM", PS1_SIM, PS1_SIM); @@ -652,4 +661,5 @@ // FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5_R3); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); # undef FORMAT_CASE @@ -732,4 +742,5 @@ SKIPPING_FORMAT ("DVO_LENSOBJ_PS1_V4", PS1_V4, PS1_V4); CONVERT_FORMAT ("DVO_LENSOBJ_PS1_V5", PS1_V5, PS1_V5_R1); + CONVERT_FORMAT ("DVO_LENSOBJ_PS1_V6", PS1_V6, PS1_V6); CONVERT_FORMAT ("DVO_LENSOBJ_PS1_V5_LOAD", PS1_V5_LOAD, PS1_V5_LOAD); SKIPPING_FORMAT ("DVO_LENSOBJ_PS1_SIM", PS1_SIM, PS1_SIM); @@ -772,4 +783,5 @@ // FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5_R1); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); # undef FORMAT_CASE @@ -840,4 +852,5 @@ SKIPPING_FORMAT ("DVO_STARPAR_PS1_V4", PS1_V4, PS1_V4); CONVERT_FORMAT ("DVO_STARPAR_PS1_V5", PS1_V5, PS1_V5); + CONVERT_FORMAT ("DVO_STARPAR_PS1_V6", PS1_V6, PS1_V6); CONVERT_FORMAT ("DVO_STARPAR_PS1_V5_LOAD", PS1_V5_LOAD, PS1_V5_LOAD); # undef CONVERT_FORMAT @@ -880,4 +893,5 @@ // FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); # undef FORMAT_CASE @@ -961,4 +975,5 @@ SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V4", PS1_V4, PS1_V4); CONVERT_FORMAT ("DVO_GALPHOT_PS1_V5", PS1_V5, PS1_V5_R1); + CONVERT_FORMAT ("DVO_GALPHOT_PS1_V6", PS1_V6, PS1_V6); CONVERT_FORMAT ("DVO_GALPHOT_PS1_V5_LOAD", PS1_V5_LOAD, PS1_V5_LOAD); SKIPPING_FORMAT ("DVO_GALPHOT_PS1_SIM", PS1_SIM, PS1_SIM); @@ -1001,4 +1016,5 @@ // FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5_R1); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); # undef FORMAT_CASE @@ -1075,4 +1091,5 @@ CONVERT_FORMAT ("DVO_IMAGE_PS1_V4", PS1_V4, PS1_V4); CONVERT_FORMAT ("DVO_IMAGE_PS1_V5", PS1_V5, PS1_V5); + CONVERT_FORMAT ("DVO_IMAGE_PS1_V6", PS1_V6, PS1_V6); CONVERT_FORMAT ("DVO_IMAGE_PS1_V5_LOAD", PS1_V5_LOAD, PS1_V5_LOAD); CONVERT_FORMAT ("DVO_IMAGE_PS1_REF", PS1_REF, PS1_REF); @@ -1096,4 +1113,5 @@ tmpImage = ImageInternalTo_##TYPE ((Image *) ftable[0].buffer, Nimage); \ free (ftable[0].buffer); \ + ftable[0].buffer = NULL; \ gfits_free_header (ftable->header); \ gfits_table_set_Image_##TYPE (ftable, tmpImage, Nimage, TRUE); \ @@ -1114,4 +1132,5 @@ FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); FORMAT_CASE (PS1_REF, PS1_REF); @@ -1166,4 +1185,5 @@ FORMAT_CASE (PS1_V4, PS1_V4); FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V6, PS1_V6); FORMAT_CASE (PS1_V5_LOAD, PS1_V5_LOAD); FORMAT_CASE (PS1_REF, PS1_REF); Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c (revision 40477) @@ -17,5 +17,6 @@ out[i].D = ave[averef].D - in[i].dD / 3600.0; out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dM = in[i].dM; out[i].dt = in[i].dt; @@ -64,5 +65,5 @@ out[i].dD = 3600.0*(ave[averef].D - in[i].D); out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].Mcal = in[i].McalPSF; out[i].dM = in[i].dM; out[i].dt = in[i].dt; @@ -182,6 +183,7 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].MpsfChp = in[i].M; + out[i].dMpsfChp = in[i].dM; + out[i].Mchisq= pow (10.0, 0.01*in[i].Xm); out[i].Ncode = in[i].Ncode; @@ -199,6 +201,7 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; + out[i].Xm = 100.0*log10(in[i].Mchisq); out[i].Ncode = in[i].Ncode; @@ -237,7 +240,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -306,7 +312,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 40477) @@ -17,5 +17,6 @@ out[i].D = ave[averef].D - in[i].dD / 3600.0; out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].Map = in[i].Map; out[i].dM = in[i].dM; @@ -62,5 +63,5 @@ out[i].dD = 3600.0*(ave[averef].D - in[i].D); out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].Mcal = in[i].McalPSF; out[i].Map = in[i].Map; out[i].dM = in[i].dM; @@ -177,6 +178,7 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].MpsfChp = in[i].M; + out[i].dMpsfChp = in[i].dM; + out[i].Mchisq= pow (10.0, 0.01*in[i].Xm); out[i].Ncode = in[i].Ncode; @@ -194,6 +196,7 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; + out[i].Xm = 100.0*log10(in[i].Mchisq); out[i].Ncode = in[i].Ncode; @@ -232,7 +235,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -298,7 +304,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_3.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_3.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_3.c (revision 40477) @@ -34,7 +34,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -95,7 +98,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c (revision 40477) @@ -114,6 +114,6 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].MpsfChp = in[i].M; + out[i].dMpsfChp = in[i].dM; } return (out); @@ -128,6 +128,6 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; } return (out); @@ -163,7 +163,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -227,7 +230,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_SIM.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_SIM.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_SIM.c (revision 40477) @@ -18,5 +18,6 @@ out[i].M = in[i].M; out[i].dM = in[i].dM; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dt = in[i].dt; out[i].airmass = in[i].airmass; @@ -64,5 +65,5 @@ out[i].M = in[i].M; out[i].dM = in[i].dM; - out[i].Mcal = in[i].Mcal; + out[i].Mcal = in[i].McalPSF; out[i].dt = in[i].dt; out[i].airmass = in[i].airmass; @@ -199,6 +200,7 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].MpsfChp = in[i].M; + out[i].dMpsfChp = in[i].dM; + out[i].Ncode = in[i].Ncode; out[i].Nused = in[i].Nused; @@ -217,6 +219,7 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; + out[i].Ncode = in[i].Ncode; out[i].Nused = in[i].Nused; @@ -320,7 +323,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -386,7 +392,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 40477) @@ -17,5 +17,6 @@ out[i].D = ave[averef].D - in[i].dD / 3600.0; out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].Map = in[i].Map; out[i].dM = in[i].dM; @@ -71,5 +72,5 @@ out[i].dD = 3600.0*(ave[averef].D - in[i].D); out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].Mcal = in[i].McalPSF; out[i].Map = in[i].Map; out[i].dM = in[i].dM; @@ -197,6 +198,7 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].MpsfChp = in[i].M; + out[i].dMpsfChp = in[i].dM; + out[i].Mchisq= in[i].Mchisq; out[i].Ncode = in[i].Ncode; @@ -216,6 +218,7 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; + out[i].Mchisq= in[i].Mchisq; out[i].Ncode = in[i].Ncode; @@ -256,7 +259,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -321,7 +327,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 40477) @@ -17,5 +17,6 @@ out[i].D = ave[averef].D - in[i].dD / 3600.0; out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].Map = in[i].Map; out[i].dM = in[i].dM; @@ -72,5 +73,5 @@ out[i].dD = 3600.0*(ave[averef].D - in[i].D); out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].Mcal = in[i].McalPSF; out[i].Map = in[i].Map; out[i].dM = in[i].dM; @@ -206,6 +207,7 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].MpsfChp = in[i].M; + out[i].dMpsfChp = in[i].dM; + out[i].Mchisq= in[i].Mchisq; out[i].flags = in[i].flags; @@ -226,6 +228,7 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; + out[i].Mchisq= in[i].Mchisq; out[i].flags = in[i].flags; @@ -267,7 +270,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -332,7 +338,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c (revision 40477) @@ -17,5 +17,6 @@ out[i].D = ave[averef].D - in[i].dD / 3600.0; out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].Map = in[i].Map; out[i].dM = in[i].dM; @@ -72,5 +73,5 @@ out[i].dD = 3600.0*(ave[averef].D - in[i].D); out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].Mcal = in[i].McalPSF; out[i].Map = in[i].Map; out[i].dM = in[i].dM; @@ -210,7 +211,9 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].Map = in[i].Map; - out[i].dM = in[i].dM; + out[i].MpsfChp = in[i].M; + out[i].sMpsfChp = in[i].Mstdev; + out[i].dMpsfChp = in[i].dM; + out[i].MapChp = in[i].Map; + out[i].Mchisq = in[i].Mchisq; out[i].flags = in[i].flags; @@ -219,5 +222,4 @@ out[i].Mmin = in[i].M_20*0.001; out[i].Mmax = in[i].M_80*0.001; - out[i].Mstdev = in[i].Mstdev; out[i].ubercalDist = in[i].ubercalDist; } @@ -233,7 +235,9 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].Map = in[i].Map; - out[i].dM = in[i].dM; + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; + out[i].Mstdev = in[i].sMpsfChp; + out[i].Map = in[i].MapChp; + out[i].Mchisq = in[i].Mchisq; out[i].flags = in[i].flags; @@ -242,5 +246,4 @@ out[i].M_20 = in[i].Mmin*1000.0; out[i].M_80 = in[i].Mmax*1000.0; - out[i].Mstdev = in[i].Mstdev; out[i].ubercalDist = in[i].ubercalDist; } @@ -277,7 +280,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -342,7 +348,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c (revision 40477) @@ -17,5 +17,6 @@ out[i].D = ave[averef].D - in[i].dD / 3600.0; out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].Map = in[i].Map; out[i].Mkron = in[i].Mkron; @@ -81,5 +82,5 @@ out[i].dD = 3600.0*(ave[averef].D - in[i].D); out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].Mcal = in[i].McalPSF; out[i].Map = in[i].Map; out[i].Mkron = in[i].Mkron; @@ -234,9 +235,10 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].Map = in[i].Map; - out[i].Mkron = in[i].Mkron; - out[i].dMkron = in[i].dMkron; - out[i].dM = in[i].dM; + out[i].MpsfChp = in[i].M; + out[i].sMpsfChp = in[i].Mstdev; + out[i].dMpsfChp = in[i].dM; + out[i].MapChp = in[i].Map; + out[i].MkronChp = in[i].Mkron; + out[i].Mchisq = in[i].Mchisq; out[i].FpsfStk = in[i].FluxPSF; @@ -249,5 +251,4 @@ out[i].Mmin = in[i].M_20*0.001; out[i].Mmax = in[i].M_80*0.001; - out[i].Mstdev = in[i].Mstdev; out[i].ubercalDist = in[i].ubercalDist; out[i].stackPrmryOff = in[i].stackPrmryOff; @@ -265,9 +266,10 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].Map = in[i].Map; - out[i].Mkron = in[i].Mkron; - out[i].dMkron = in[i].dMkron; - out[i].dM = in[i].dM; + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; + out[i].Mstdev = in[i].sMpsfChp; + out[i].Map = in[i].MapChp; + out[i].Mkron = in[i].MkronChp; + out[i].Mchisq = in[i].Mchisq; out[i].FluxPSF = in[i].FpsfStk; @@ -280,5 +282,4 @@ out[i].M_20 = in[i].Mmin*1000.0; out[i].M_80 = in[i].Mmax*1000.0; - out[i].Mstdev = in[i].Mstdev; out[i].ubercalDist = in[i].ubercalDist; out[i].stackPrmryOff = in[i].stackPrmryOff; @@ -317,7 +318,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -382,7 +386,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -513,5 +519,6 @@ out[i].D = ave[averef].D - in[i].dD / 3600.0; out[i].M = in[i].M; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].Map = in[i].Map; out[i].Mkron = in[i].Mkron; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c (revision 40477) @@ -22,5 +22,6 @@ out[i].Mkron = in[i].Mkron; out[i].dMkron = in[i].dMkron; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; out[i].dt = in[i].dt; @@ -96,5 +97,5 @@ out[i].Mkron = in[i].Mkron; out[i].dMkron = in[i].dMkron; - out[i].Mcal = in[i].Mcal; + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; out[i].dt = in[i].dt; @@ -311,17 +312,17 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].dM = in[i].dM; - out[i].Map = in[i].Map; - out[i].dMap = in[i].dMap; - out[i].sMap = in[i].sMap; - out[i].Mkron = in[i].Mkron; - out[i].dMkron = in[i].dMkron; - out[i].sMkron = in[i].sMkron; + out[i].MpsfChp = in[i].M; + out[i].sMpsfChp = in[i].Mstdev; + out[i].dMpsfChp = in[i].dM; + out[i].MapChp = in[i].Map; + out[i].dMapChp = in[i].dMap; + out[i].sMapChp = in[i].sMap; + out[i].MkronChp = in[i].Mkron; + out[i].dMkronChp = in[i].dMkron; + out[i].sMkronChp = in[i].sMkron; out[i].psfQfMax = in[i].psfQfMax; out[i].psfQfPerfMax = in[i].psfQfPerfMax; - out[i].Mstdev = in[i].Mstdev; out[i].Mmin = in[i].Mmin; out[i].Mmax = in[i].Mmax; @@ -389,17 +390,17 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].dM = in[i].dM; - out[i].Map = in[i].Map; - out[i].dMap = in[i].dMap; - out[i].sMap = in[i].sMap; - out[i].Mkron = in[i].Mkron; - out[i].dMkron = in[i].dMkron; - out[i].sMkron = in[i].sMkron; + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; + out[i].Mstdev = in[i].sMpsfChp; + out[i].Map = in[i].MapChp; + out[i].dMap = in[i].dMapChp; + out[i].sMap = in[i].sMapChp; + out[i].Mkron = in[i].MkronChp; + out[i].dMkron = in[i].dMkronChp; + out[i].sMkron = in[i].sMkronChp; out[i].psfQfMax = in[i].psfQfMax; out[i].psfQfPerfMax = in[i].psfQfPerfMax; - out[i].Mstdev = in[i].Mstdev; out[i].Mmin = in[i].Mmin; out[i].Mmax = in[i].Mmax; @@ -1143,7 +1144,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -1209,7 +1213,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -1495,13 +1501,14 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].dM = in[i].dM; - out[i].Map = in[i].Map; - out[i].dMap = in[i].dMap; - out[i].sMap = in[i].sMap; - out[i].Mkron = in[i].Mkron; - out[i].dMkron = in[i].dMkron; - - out[i].Mstdev = in[i].Mstdev; + out[i].MpsfChp = in[i].M; + out[i].dMpsfChp = in[i].dM; + out[i].sMpsfChp = in[i].Mstdev; + out[i].MapChp = in[i].Map; + out[i].dMapChp = in[i].dMap; + out[i].sMapChp = in[i].sMap; + out[i].MkronChp = in[i].Mkron; + out[i].dMkronChp = in[i].dMkron; + out[i].sMkronChp = in[i].sMkron; + out[i].Mmin = in[i].Mmin; out[i].Mmax = in[i].Mmax; @@ -1668,5 +1675,6 @@ out[i].Mkron = in[i].Mkron; out[i].dMkron = in[i].dMkron; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; out[i].dt = in[i].dt; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V5_LOAD.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V5_LOAD.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V5_LOAD.c (revision 40477) @@ -22,5 +22,6 @@ out[i].Mkron = in[i].Mkron; out[i].dMkron = in[i].dMkron; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; out[i].dt = in[i].dt; @@ -96,5 +97,5 @@ out[i].Mkron = in[i].Mkron; out[i].dMkron = in[i].dMkron; - out[i].Mcal = in[i].Mcal; + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; out[i].dt = in[i].dt; @@ -221,5 +222,5 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; + out[i].MpsfChp = in[i].M; out[i].Ncode = in[i].Ncode; out[i].Nused = in[i].Nused; @@ -237,5 +238,5 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; + out[i].M = in[i].MpsfChp; out[i].Ncode = in[i].Ncode; out[i].Nused = in[i].Nused; @@ -654,7 +655,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -720,7 +724,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V6.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V6.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_PS1_V6.c (revision 40477) @@ -0,0 +1,1054 @@ +# include + +/* convert PS1_V6 formats to internal formats */ + +Measure *Measure_PS1_V6_ToInternal (Average *ave, Measure_PS1_V6 *in, off_t Nvalues) { + OHANA_UNUSED_PARAM(ave); + + off_t i; + Measure *out; + + ALLOCATE_ZERO (out, Measure, Nvalues); + + for (i = 0; i < Nvalues; i++) { + dvo_measure_init (&out[i]); + + out[i].R = in[i].R; + out[i].D = in[i].D; + out[i].M = in[i].M; + out[i].dM = in[i].dM; + out[i].Map = in[i].Map; + out[i].dMap = in[i].dMap; + out[i].Mkron = in[i].Mkron; + out[i].dMkron = in[i].dMkron; + out[i].McalPSF = in[i].McalPSF; + out[i].McalAPER = in[i].McalAPER; + out[i].dMcal = in[i].dMcal; + out[i].dt = in[i].dt; + out[i].FluxPSF = in[i].FluxPSF; + out[i].dFluxPSF = in[i].dFluxPSF; + out[i].FluxKron = in[i].FluxKron; + out[i].dFluxKron = in[i].dFluxKron; + out[i].FluxAp = in[i].FluxAp; + out[i].dFluxAp = in[i].dFluxAp; + out[i].airmass = in[i].airmass; + out[i].az = in[i].az; + out[i].Xccd = in[i].Xccd; + out[i].Yccd = in[i].Yccd; + out[i].Xfix = in[i].Xfix; + out[i].Yfix = in[i].Yfix; + out[i].XoffKH = in[i].XoffKH; + out[i].YoffKH = in[i].YoffKH; + out[i].XoffDCR = in[i].XoffDCR; + out[i].YoffDCR = in[i].YoffDCR; + out[i].XoffCAM = in[i].XoffCAM; + out[i].YoffCAM = in[i].YoffCAM; + out[i].Mflat = in[i].Mflat; + out[i].Sky = in[i].Sky; + out[i].dSky = in[i].dSky; + out[i].t = in[i].t; + out[i].averef = in[i].averef; + out[i].detID = in[i].detID; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].extID = in[i].extID; + out[i].imageID = in[i].imageID; + out[i].psfQF = in[i].psfQF; + out[i].psfQFperf = in[i].psfQFperf; + out[i].psfChisq = in[i].psfChisq; + out[i].psfNdof = in[i].psfNdof; + out[i].psfNpix = in[i].psfNpix; + out[i].extNsigma = in[i].extNsigma; + out[i].FWx = in[i].FWx; + out[i].FWy = in[i].FWy; + out[i].theta = in[i].theta; + out[i].Mxx = in[i].Mxx; + out[i].Mxy = in[i].Mxy; + out[i].Myy = in[i].Myy; + out[i].t_msec = in[i].t_msec; + out[i].photcode = in[i].photcode; + out[i].dXccd = in[i].dXccd; + out[i].dYccd = in[i].dYccd; + out[i].dRsys = in[i].dRsys; + out[i].posangle = in[i].posangle; + out[i].pltscale = in[i].pltscale; + out[i].dbFlags = in[i].dbFlags; + out[i].photFlags = in[i].photFlags; + out[i].photFlags2 = in[i].photFlags2; + } + return (out); +} + +Measure_PS1_V6 *MeasureInternalTo_PS1_V6 (Average *ave, Measure *in, off_t Nvalues) { + OHANA_UNUSED_PARAM(ave); + + off_t i; + Measure_PS1_V6 *out; + + ALLOCATE_ZERO (out, Measure_PS1_V6, Nvalues); + + for (i = 0; i < Nvalues; i++) { + out[i].R = in[i].R; + out[i].D = in[i].D; + out[i].M = in[i].M; + out[i].dM = in[i].dM; + out[i].Map = in[i].Map; + out[i].dMap = in[i].dMap; + out[i].Mkron = in[i].Mkron; + out[i].dMkron = in[i].dMkron; + out[i].McalPSF = in[i].McalPSF; + out[i].McalAPER = in[i].McalAPER; + out[i].dMcal = in[i].dMcal; + out[i].dt = in[i].dt; + out[i].FluxPSF = in[i].FluxPSF; + out[i].dFluxPSF = in[i].dFluxPSF; + out[i].FluxKron = in[i].FluxKron; + out[i].dFluxKron = in[i].dFluxKron; + out[i].FluxAp = in[i].FluxAp; + out[i].dFluxAp = in[i].dFluxAp; + out[i].airmass = in[i].airmass; + out[i].az = in[i].az; + out[i].Xccd = in[i].Xccd; + out[i].Yccd = in[i].Yccd; + out[i].Xfix = in[i].Xfix; + out[i].Yfix = in[i].Yfix; + out[i].XoffKH = in[i].XoffKH; + out[i].YoffKH = in[i].YoffKH; + out[i].XoffDCR = in[i].XoffDCR; + out[i].YoffDCR = in[i].YoffDCR; + out[i].XoffCAM = in[i].XoffCAM; + out[i].YoffCAM = in[i].YoffCAM; + out[i].Mflat = in[i].Mflat; + out[i].Sky = in[i].Sky; + out[i].dSky = in[i].dSky; + out[i].t = in[i].t; + out[i].averef = in[i].averef; + out[i].detID = in[i].detID; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].extID = in[i].extID; + out[i].imageID = in[i].imageID; + out[i].psfQF = in[i].psfQF; + out[i].psfQFperf = in[i].psfQFperf; + out[i].psfChisq = in[i].psfChisq; + out[i].psfNdof = in[i].psfNdof; + out[i].psfNpix = in[i].psfNpix; + out[i].extNsigma = in[i].extNsigma; + out[i].FWx = in[i].FWx; + out[i].FWy = in[i].FWy; + out[i].theta = in[i].theta; + out[i].Mxx = in[i].Mxx; + out[i].Mxy = in[i].Mxy; + out[i].Myy = in[i].Myy; + out[i].t_msec = in[i].t_msec; + out[i].photcode = in[i].photcode; + out[i].dXccd = in[i].dXccd; + out[i].dYccd = in[i].dYccd; + out[i].dRsys = in[i].dRsys; + out[i].posangle = in[i].posangle; + out[i].pltscale = in[i].pltscale; + out[i].dbFlags = in[i].dbFlags; + out[i].photFlags = in[i].photFlags; + out[i].photFlags2 = in[i].photFlags2; + } + return (out); +} + +// 'primary' is needed to conform with the API for Loneos and Elixir, but is not used +Average *Average_PS1_V6_ToInternal (Average_PS1_V6 *in, off_t Nvalues, SecFilt **primary) { + OHANA_UNUSED_PARAM(primary); + + off_t i; + Average *out; + + ALLOCATE_ZERO (out, Average, Nvalues); + + for (i = 0; i < Nvalues; i++) { + dvo_average_init (&out[i]); + + out[i].R = in[i].R; + out[i].D = in[i].D; + out[i].dR = in[i].dR; + out[i].dD = in[i].dD; + out[i].uR = in[i].uR; + out[i].uD = in[i].uD; + out[i].duR = in[i].duR; + out[i].duD = in[i].duD; + out[i].P = in[i].P; + out[i].dP = in[i].dP; + + out[i].Rstk = in[i].Rstk; + out[i].Dstk = in[i].Dstk; + out[i].dRstk = in[i].dRstk; + out[i].dDstk = in[i].dDstk; + + out[i].ChiSqAve = in[i].ChiSqAve; + out[i].ChiSqPM = in[i].ChiSqPM; + out[i].ChiSqPar = in[i].ChiSqPar; + out[i].Tmean = in[i].Tmean; + out[i].Trange = in[i].Trange; + + out[i].psfQF = in[i].psfQF; + out[i].psfQFperf = in[i].psfQFperf; + out[i].stargal = in[i].stargal; + out[i].Npos = in[i].Npos; + + out[i].Nmeasure = in[i].Nmeasure; + out[i].Nmissing = in[i].Nmissing; + out[i].Nlensing = in[i].Nlensing; + out[i].Nlensobj = in[i].Nlensobj; + out[i].Nstarpar = in[i].Nstarpar; + out[i].Ngalphot = in[i].Ngalphot; + + out[i].measureOffset = in[i].measureOffset; + out[i].missingOffset = in[i].missingOffset; + out[i].lensingOffset = in[i].lensingOffset; + out[i].lensobjOffset = in[i].lensobjOffset; + out[i].starparOffset = in[i].starparOffset; + out[i].galphotOffset = in[i].galphotOffset; + + out[i].refColorBlue = in[i].refColorBlue; + out[i].refColorRed = in[i].refColorRed; + + out[i].tessID = in[i].tessID; + out[i].skycellID = in[i].skycellID; + out[i].projectionID = in[i].projectionID; + + out[i].NwarpOK = in[i].NwarpOK; + + out[i].flags = in[i].flags; + out[i].photFlagsUpper = in[i].photFlagsUpper; + out[i].photFlagsLower = in[i].photFlagsLower; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].extID = in[i].extID; + + out[i].uRgal = in[i].uRgal; + out[i].uDgal = in[i].uDgal; + } + return (out); +} + +// 'primary' is needed to conform with the API for Loneos and Elixir, but is not used +Average_PS1_V6 *AverageInternalTo_PS1_V6 (Average *in, off_t Nvalues, SecFilt *primary) { + OHANA_UNUSED_PARAM(primary); + + off_t i; + Average_PS1_V6 *out; + + ALLOCATE_ZERO (out, Average_PS1_V6, Nvalues); + + for (i = 0; i < Nvalues; i++) { + out[i].R = in[i].R; + out[i].D = in[i].D; + out[i].dR = in[i].dR; + out[i].dD = in[i].dD; + out[i].uR = in[i].uR; + out[i].uD = in[i].uD; + out[i].duR = in[i].duR; + out[i].duD = in[i].duD; + out[i].P = in[i].P; + out[i].dP = in[i].dP; + + out[i].Rstk = in[i].Rstk; + out[i].Dstk = in[i].Dstk; + out[i].dRstk = in[i].dRstk; + out[i].dDstk = in[i].dDstk; + + out[i].ChiSqAve = in[i].ChiSqAve; + out[i].ChiSqPM = in[i].ChiSqPM; + out[i].ChiSqPar = in[i].ChiSqPar; + out[i].Tmean = in[i].Tmean; + out[i].Trange = in[i].Trange; + + out[i].psfQF = in[i].psfQF; + out[i].psfQFperf = in[i].psfQFperf; + out[i].stargal = in[i].stargal; + out[i].Npos = in[i].Npos; + + out[i].Nmeasure = in[i].Nmeasure; + out[i].Nmissing = in[i].Nmissing; + out[i].Nlensing = in[i].Nlensing; + out[i].Nlensobj = in[i].Nlensobj; + out[i].Nstarpar = in[i].Nstarpar; + out[i].Ngalphot = in[i].Ngalphot; + + out[i].measureOffset = in[i].measureOffset; + out[i].missingOffset = in[i].missingOffset; + out[i].lensingOffset = in[i].lensingOffset; + out[i].lensobjOffset = in[i].lensobjOffset; + out[i].starparOffset = in[i].starparOffset; + out[i].galphotOffset = in[i].galphotOffset; + + out[i].refColorBlue = in[i].refColorBlue; + out[i].refColorRed = in[i].refColorRed; + + out[i].tessID = in[i].tessID; + out[i].skycellID = in[i].skycellID; + out[i].projectionID = in[i].projectionID; + + out[i].NwarpOK = in[i].NwarpOK; + + out[i].flags = in[i].flags; + out[i].photFlagsUpper = in[i].photFlagsUpper; + out[i].photFlagsLower = in[i].photFlagsLower; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].extID = in[i].extID; + + out[i].uRgal = in[i].uRgal; + out[i].uDgal = in[i].uDgal; + } + return (out); +} + +SecFilt *SecFilt_PS1_V6_ToInternal (SecFilt_PS1_V6 *in, off_t Nvalues) { + + off_t i; + SecFilt *out; + + ALLOCATE_ZERO (out, SecFilt, Nvalues); + + for (i = 0; i < Nvalues; i++) { + dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); + + out[i].MpsfChp = in[i].M; + out[i].sMpsfChp = in[i].Mstdev; + out[i].dMpsfChp = in[i].dM; + out[i].MapChp = in[i].Map; + out[i].dMapChp = in[i].dMap; + out[i].sMapChp = in[i].sMap; + out[i].MkronChp = in[i].Mkron; + out[i].dMkronChp = in[i].dMkron; + out[i].sMkronChp = in[i].sMkron; + + out[i].psfQfMax = in[i].psfQfMax; + out[i].psfQfPerfMax = in[i].psfQfPerfMax; + + out[i].Mmin = in[i].Mmin; + out[i].Mmax = in[i].Mmax; + out[i].Mchisq = in[i].Mchisq; + + out[i].Ncode = in[i].Ncode; + out[i].Nused = in[i].Nused; + out[i].NusedKron = in[i].NusedKron; + out[i].NusedAp = in[i].NusedAp; + + out[i].flags = in[i].flags; + + out[i].MpsfStk = in[i].MpsfStk; + out[i].FpsfStk = in[i].FpsfStk; + out[i].dFpsfStk = in[i].dFpsfStk; + + out[i].MkronStk = in[i].MkronStk; + out[i].FkronStk = in[i].FkronStk; + out[i].dFkronStk = in[i].dFkronStk; + + out[i].MapStk = in[i].MapStk; + out[i].FapStk = in[i].FapStk; + out[i].dFapStk = in[i].dFapStk; + + out[i].Nstack = in[i].Nstack; + out[i].NstackDet = in[i].NstackDet; + + out[i].stackPrmryOff = in[i].stackPrmryOff; + out[i].stackBestOff = in[i].stackBestOff; + + out[i].MpsfWrp = in[i].MpsfWrp; + out[i].FpsfWrp = in[i].FpsfWrp; + out[i].dFpsfWrp = in[i].dFpsfWrp; + out[i].sFpsfWrp = in[i].sFpsfWrp; + + out[i].MkronWrp = in[i].MkronWrp; + out[i].FkronWrp = in[i].FkronWrp; + out[i].dFkronWrp = in[i].dFkronWrp; + out[i].sFkronWrp = in[i].sFkronWrp; + + out[i].MapWrp = in[i].MapWrp; + out[i].FapWrp = in[i].FapWrp; + out[i].dFapWrp = in[i].dFapWrp; + out[i].sFapWrp = in[i].sFapWrp; + + out[i].NusedWrp = in[i].NusedWrp; + out[i].NusedKronWrp = in[i].NusedKronWrp; + out[i].NusedApWrp = in[i].NusedApWrp; + + out[i].Nwarp = in[i].Nwarp; + out[i].NwarpGood = in[i].NwarpGood; + + out[i].ubercalDist = in[i].ubercalDist; + } + return (out); +} + +SecFilt_PS1_V6 *SecFiltInternalTo_PS1_V6 (SecFilt *in, off_t Nvalues) { + + off_t i; + SecFilt_PS1_V6 *out; + + ALLOCATE_ZERO (out, SecFilt_PS1_V6, Nvalues); + + for (i = 0; i < Nvalues; i++) { + + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; + out[i].Mstdev = in[i].sMpsfChp; + out[i].Map = in[i].MapChp; + out[i].dMap = in[i].dMapChp; + out[i].sMap = in[i].sMapChp; + out[i].Mkron = in[i].MkronChp; + out[i].dMkron = in[i].dMkronChp; + out[i].sMkron = in[i].sMkronChp; + + out[i].psfQfMax = in[i].psfQfMax; + out[i].psfQfPerfMax = in[i].psfQfPerfMax; + + out[i].Mmin = in[i].Mmin; + out[i].Mmax = in[i].Mmax; + out[i].Mchisq = in[i].Mchisq; + + out[i].Ncode = in[i].Ncode; + out[i].Nused = in[i].Nused; + out[i].NusedKron = in[i].NusedKron; + out[i].NusedAp = in[i].NusedAp; + + out[i].flags = in[i].flags; + + out[i].MpsfStk = in[i].MpsfStk; + out[i].FpsfStk = in[i].FpsfStk; + out[i].dFpsfStk = in[i].dFpsfStk; + + out[i].MkronStk = in[i].MkronStk; + out[i].FkronStk = in[i].FkronStk; + out[i].dFkronStk = in[i].dFkronStk; + + out[i].MapStk = in[i].MapStk; + out[i].FapStk = in[i].FapStk; + out[i].dFapStk = in[i].dFapStk; + + out[i].Nstack = in[i].Nstack; + out[i].NstackDet = in[i].NstackDet; + + out[i].stackPrmryOff = in[i].stackPrmryOff; + out[i].stackBestOff = in[i].stackBestOff; + + out[i].MpsfWrp = in[i].MpsfWrp; + out[i].FpsfWrp = in[i].FpsfWrp; + out[i].dFpsfWrp = in[i].dFpsfWrp; + out[i].sFpsfWrp = in[i].sFpsfWrp; + + out[i].MkronWrp = in[i].MkronWrp; + out[i].FkronWrp = in[i].FkronWrp; + out[i].dFkronWrp = in[i].dFkronWrp; + out[i].sFkronWrp = in[i].sFkronWrp; + + out[i].MapWrp = in[i].MapWrp; + out[i].FapWrp = in[i].FapWrp; + out[i].dFapWrp = in[i].dFapWrp; + out[i].sFapWrp = in[i].sFapWrp; + + out[i].NusedWrp = in[i].NusedWrp; + out[i].NusedKronWrp = in[i].NusedKronWrp; + out[i].NusedApWrp = in[i].NusedApWrp; + + out[i].Nwarp = in[i].Nwarp; + out[i].NwarpGood = in[i].NwarpGood; + + out[i].ubercalDist = in[i].ubercalDist; + } + return (out); +} + +Lensing *Lensing_PS1_V6_ToInternal (Lensing_PS1_V6 *in, off_t Nvalues) { + + off_t i; + Lensing *out; + + ALLOCATE_ZERO (out, Lensing, Nvalues); + + for (i = 0; i < Nvalues; i++) { + dvo_lensing_init (&out[i]); + + out[i].X11_sm_obj = in[i].X11_sm_obj; + out[i].X12_sm_obj = in[i].X12_sm_obj; + out[i].X22_sm_obj = in[i].X22_sm_obj; + out[i].E1_sm_obj = in[i].E1_sm_obj; + out[i].E2_sm_obj = in[i].E2_sm_obj; + + out[i].X11_sh_obj = in[i].X11_sh_obj; + out[i].X12_sh_obj = in[i].X12_sh_obj; + out[i].X22_sh_obj = in[i].X22_sh_obj; + out[i].E1_sh_obj = in[i].E1_sh_obj; + out[i].E2_sh_obj = in[i].E2_sh_obj; + + out[i].X11_sm_psf = in[i].X11_sm_psf; + out[i].X12_sm_psf = in[i].X12_sm_psf; + out[i].X22_sm_psf = in[i].X22_sm_psf; + out[i].E1_sm_psf = in[i].E1_sm_psf; + out[i].E2_sm_psf = in[i].E2_sm_psf; + + out[i].X11_sh_psf = in[i].X11_sh_psf; + out[i].X12_sh_psf = in[i].X12_sh_psf; + out[i].X22_sh_psf = in[i].X22_sh_psf; + out[i].E1_sh_psf = in[i].E1_sh_psf; + out[i].E2_sh_psf = in[i].E2_sh_psf; + + out[i].E1_psf = in[i].E1_psf; + out[i].E2_psf = in[i].E2_psf; + + out[i].F_ApR5 = in[i].F_ApR5; + out[i].dF_ApR5 = in[i].dF_ApR5; + out[i].sF_ApR5 = in[i].sF_ApR5; + out[i].fF_ApR5 = in[i].fF_ApR5; + + out[i].F_ApR6 = in[i].F_ApR6; + out[i].dF_ApR6 = in[i].dF_ApR6; + out[i].sF_ApR6 = in[i].sF_ApR6; + out[i].fF_ApR6 = in[i].fF_ApR6; + + out[i].F_ApR7 = in[i].F_ApR7; + out[i].dF_ApR7 = in[i].dF_ApR7; + out[i].sF_ApR7 = in[i].sF_ApR7; + out[i].fF_ApR7 = in[i].fF_ApR7; + + out[i].detID = in[i].detID; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].averef = in[i].averef; + + out[i].imageID = in[i].imageID; + out[i].oldImID = in[i].oldImID; + } + return (out); +} + +Lensing_PS1_V6 *LensingInternalTo_PS1_V6 (Lensing *in, off_t Nvalues) { + + off_t i; + Lensing_PS1_V6 *out; + + ALLOCATE_ZERO (out, Lensing_PS1_V6, Nvalues); + + for (i = 0; i < Nvalues; i++) { + + out[i].X11_sm_obj = in[i].X11_sm_obj; + out[i].X12_sm_obj = in[i].X12_sm_obj; + out[i].X22_sm_obj = in[i].X22_sm_obj; + out[i].E1_sm_obj = in[i].E1_sm_obj; + out[i].E2_sm_obj = in[i].E2_sm_obj; + + out[i].X11_sh_obj = in[i].X11_sh_obj; + out[i].X12_sh_obj = in[i].X12_sh_obj; + out[i].X22_sh_obj = in[i].X22_sh_obj; + out[i].E1_sh_obj = in[i].E1_sh_obj; + out[i].E2_sh_obj = in[i].E2_sh_obj; + + out[i].X11_sm_psf = in[i].X11_sm_psf; + out[i].X12_sm_psf = in[i].X12_sm_psf; + out[i].X22_sm_psf = in[i].X22_sm_psf; + out[i].E1_sm_psf = in[i].E1_sm_psf; + out[i].E2_sm_psf = in[i].E2_sm_psf; + + out[i].X11_sh_psf = in[i].X11_sh_psf; + out[i].X12_sh_psf = in[i].X12_sh_psf; + out[i].X22_sh_psf = in[i].X22_sh_psf; + out[i].E1_sh_psf = in[i].E1_sh_psf; + out[i].E2_sh_psf = in[i].E2_sh_psf; + + out[i].E1_psf = in[i].E1_psf; + out[i].E2_psf = in[i].E2_psf; + + out[i].F_ApR5 = in[i].F_ApR5; + out[i].dF_ApR5 = in[i].dF_ApR5; + out[i].sF_ApR5 = in[i].sF_ApR5; + out[i].fF_ApR5 = in[i].fF_ApR5; + + out[i].F_ApR6 = in[i].F_ApR6; + out[i].dF_ApR6 = in[i].dF_ApR6; + out[i].sF_ApR6 = in[i].sF_ApR6; + out[i].fF_ApR6 = in[i].fF_ApR6; + + out[i].F_ApR7 = in[i].F_ApR7; + out[i].dF_ApR7 = in[i].dF_ApR7; + out[i].sF_ApR7 = in[i].sF_ApR7; + out[i].fF_ApR7 = in[i].fF_ApR7; + + out[i].detID = in[i].detID; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].averef = in[i].averef; + + out[i].imageID = in[i].imageID; + out[i].oldImID = in[i].oldImID; + } + return (out); +} + +Lensobj *Lensobj_PS1_V6_ToInternal (Lensobj_PS1_V6 *in, off_t Nvalues) { + + off_t i; + Lensobj *out; + + ALLOCATE_ZERO (out, Lensobj, Nvalues); + + for (i = 0; i < Nvalues; i++) { + dvo_lensobj_init (&out[i], FALSE); + + out[i].X11_sm_obj = in[i].X11_sm_obj; + out[i].X12_sm_obj = in[i].X12_sm_obj; + out[i].X22_sm_obj = in[i].X22_sm_obj; + out[i].E1_sm_obj = in[i].E1_sm_obj; + out[i].E2_sm_obj = in[i].E2_sm_obj; + + out[i].X11_sh_obj = in[i].X11_sh_obj; + out[i].X12_sh_obj = in[i].X12_sh_obj; + out[i].X22_sh_obj = in[i].X22_sh_obj; + out[i].E1_sh_obj = in[i].E1_sh_obj; + out[i].E2_sh_obj = in[i].E2_sh_obj; + + out[i].X11_sm_psf = in[i].X11_sm_psf; + out[i].X12_sm_psf = in[i].X12_sm_psf; + out[i].X22_sm_psf = in[i].X22_sm_psf; + out[i].E1_sm_psf = in[i].E1_sm_psf; + out[i].E2_sm_psf = in[i].E2_sm_psf; + + out[i].X11_sh_psf = in[i].X11_sh_psf; + out[i].X12_sh_psf = in[i].X12_sh_psf; + out[i].X22_sh_psf = in[i].X22_sh_psf; + out[i].E1_sh_psf = in[i].E1_sh_psf; + out[i].E2_sh_psf = in[i].E2_sh_psf; + + out[i].F_ApR5 = in[i].F_ApR5; + out[i].dF_ApR5 = in[i].dF_ApR5; + out[i].sF_ApR5 = in[i].sF_ApR5; + out[i].fF_ApR5 = in[i].fF_ApR5; + + out[i].F_ApR6 = in[i].F_ApR6; + out[i].dF_ApR6 = in[i].dF_ApR6; + out[i].sF_ApR6 = in[i].sF_ApR6; + out[i].fF_ApR6 = in[i].fF_ApR6; + + out[i].F_ApR7 = in[i].F_ApR7; + out[i].dF_ApR7 = in[i].dF_ApR7; + out[i].sF_ApR7 = in[i].sF_ApR7; + out[i].fF_ApR7 = in[i].fF_ApR7; + + out[i].gamma = in[i].gamma; + out[i].E1 = in[i].E1; + out[i].E2 = in[i].E2; + + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + + out[i].photcode = in[i].photcode; + out[i].Nmeas = in[i].Nmeas; + } + return (out); +} + +Lensobj_PS1_V6 *LensobjInternalTo_PS1_V6 (Lensobj *in, off_t Nvalues) { + + off_t i; + Lensobj_PS1_V6 *out; + + ALLOCATE_ZERO (out, Lensobj_PS1_V6, Nvalues); + + for (i = 0; i < Nvalues; i++) { + + out[i].X11_sm_obj = in[i].X11_sm_obj; + out[i].X12_sm_obj = in[i].X12_sm_obj; + out[i].X22_sm_obj = in[i].X22_sm_obj; + out[i].E1_sm_obj = in[i].E1_sm_obj; + out[i].E2_sm_obj = in[i].E2_sm_obj; + + out[i].X11_sh_obj = in[i].X11_sh_obj; + out[i].X12_sh_obj = in[i].X12_sh_obj; + out[i].X22_sh_obj = in[i].X22_sh_obj; + out[i].E1_sh_obj = in[i].E1_sh_obj; + out[i].E2_sh_obj = in[i].E2_sh_obj; + + out[i].X11_sm_psf = in[i].X11_sm_psf; + out[i].X12_sm_psf = in[i].X12_sm_psf; + out[i].X22_sm_psf = in[i].X22_sm_psf; + out[i].E1_sm_psf = in[i].E1_sm_psf; + out[i].E2_sm_psf = in[i].E2_sm_psf; + + out[i].X11_sh_psf = in[i].X11_sh_psf; + out[i].X12_sh_psf = in[i].X12_sh_psf; + out[i].X22_sh_psf = in[i].X22_sh_psf; + out[i].E1_sh_psf = in[i].E1_sh_psf; + out[i].E2_sh_psf = in[i].E2_sh_psf; + + out[i].F_ApR5 = in[i].F_ApR5; + out[i].dF_ApR5 = in[i].dF_ApR5; + out[i].sF_ApR5 = in[i].sF_ApR5; + out[i].fF_ApR5 = in[i].fF_ApR5; + + out[i].F_ApR6 = in[i].F_ApR6; + out[i].dF_ApR6 = in[i].dF_ApR6; + out[i].sF_ApR6 = in[i].sF_ApR6; + out[i].fF_ApR6 = in[i].fF_ApR6; + + out[i].F_ApR7 = in[i].F_ApR7; + out[i].dF_ApR7 = in[i].dF_ApR7; + out[i].sF_ApR7 = in[i].sF_ApR7; + out[i].fF_ApR7 = in[i].fF_ApR7; + + out[i].gamma = in[i].gamma; + out[i].E1 = in[i].E1; + out[i].E2 = in[i].E2; + + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + + out[i].photcode = in[i].photcode; + out[i].Nmeas = in[i].Nmeas; + } + return (out); +} + +StarPar *StarPar_PS1_V6_ToInternal (StarPar_PS1_V6 *in, off_t Nvalues) { + + off_t i; + StarPar *out; + + ALLOCATE_ZERO (out, StarPar, Nvalues); + + for (i = 0; i < Nvalues; i++) { + dvo_starpar_init (&out[i]); + + out[i].R = in[i].R; + out[i].D = in[i].D; + out[i].galLat = in[i].galLat; + out[i].galLon = in[i].galLon; + + out[i].Ebv = in[i].Ebv ; + out[i].dEbv = in[i].dEbv ; + out[i].DistMag = in[i].DistMag ; + out[i].dDistMag = in[i].dDistMag; + out[i].M_r = in[i].M_r ; + out[i].dM_r = in[i].dM_r ; + out[i].FeH = in[i].FeH ; + out[i].dFeH = in[i].dFeH ; + out[i].uRA = in[i].uRA ; + out[i].uDEC = in[i].uDEC ; + + out[i].averef = in[i].averef; + out[i].objID = in[i].objID ; + out[i].catID = in[i].catID ; + } + return (out); +} + +StarPar_PS1_V6 *StarParInternalTo_PS1_V6 (StarPar *in, off_t Nvalues) { + + off_t i; + StarPar_PS1_V6 *out; + + ALLOCATE_ZERO (out, StarPar_PS1_V6, Nvalues); + + for (i = 0; i < Nvalues; i++) { + + out[i].R = in[i].R; + out[i].D = in[i].D; + out[i].galLat = in[i].galLat; + out[i].galLon = in[i].galLon; + + out[i].Ebv = in[i].Ebv ; + out[i].dEbv = in[i].dEbv ; + out[i].DistMag = in[i].DistMag ; + out[i].dDistMag = in[i].dDistMag; + out[i].M_r = in[i].M_r ; + out[i].dM_r = in[i].dM_r ; + out[i].FeH = in[i].FeH ; + out[i].dFeH = in[i].dFeH ; + out[i].uRA = in[i].uRA ; + out[i].uDEC = in[i].uDEC ; + + out[i].averef = in[i].averef; + out[i].objID = in[i].objID ; + out[i].catID = in[i].catID ; + } + return (out); +} + +GalPhot *GalPhot_PS1_V6_ToInternal (GalPhot_PS1_V6 *in, off_t Nvalues) { + + off_t i; + GalPhot *out; + + ALLOCATE_ZERO (out, GalPhot, Nvalues); + + for (i = 0; i < Nvalues; i++) { + dvo_galphot_init (&out[i]); + + out[i].Xfit = in[i].Xfit; + out[i].Yfit = in[i].Yfit; + out[i].mag = in[i].mag; + out[i].magErr = in[i].magErr; + out[i].majorAxis = in[i].majorAxis; + out[i].minorAxis = in[i].minorAxis; + out[i].majorAxisErr = in[i].majorAxisErr; + out[i].minorAxisErr = in[i].minorAxisErr; + out[i].theta = in[i].theta; + out[i].thetaErr = in[i].thetaErr; + out[i].index = in[i].index; + out[i].chisq = in[i].chisq; + out[i].Npix = in[i].Npix; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].detID = in[i].detID; + out[i].imageID = in[i].imageID; + out[i].averef = in[i].averef; + out[i].flags = in[i].flags; + out[i].photcode = in[i].photcode; + out[i].modelType = in[i].modelType; + } + return (out); +} + +GalPhot_PS1_V6 *GalPhotInternalTo_PS1_V6 (GalPhot *in, off_t Nvalues) { + + off_t i; + GalPhot_PS1_V6 *out; + + ALLOCATE_ZERO (out, GalPhot_PS1_V6, Nvalues); + + for (i = 0; i < Nvalues; i++) { + out[i].Xfit = in[i].Xfit; + out[i].Yfit = in[i].Yfit; + out[i].mag = in[i].mag; + out[i].magErr = in[i].magErr; + out[i].majorAxis = in[i].majorAxis; + out[i].minorAxis = in[i].minorAxis; + out[i].majorAxisErr = in[i].majorAxisErr; + out[i].minorAxisErr = in[i].minorAxisErr; + out[i].theta = in[i].theta; + out[i].thetaErr = in[i].thetaErr; + out[i].index = in[i].index; + out[i].chisq = in[i].chisq; + out[i].Npix = in[i].Npix; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].detID = in[i].detID; + out[i].imageID = in[i].imageID; + out[i].averef = in[i].averef; + out[i].flags = in[i].flags; + out[i].photcode = in[i].photcode; + out[i].modelType = in[i].modelType; + } + return (out); +} + +# define RAW_IMAGE_NAME_LEN 117 + +Image *Image_PS1_V6_ToInternal (Image_PS1_V6 *in, off_t Nvalues, off_t Nalloc) { + + off_t i; + Image *out; + + char *buffer; + ALLOCATE_ZERO (buffer, char, Nalloc); + out = (Image *) buffer; + // ALLOCATE_ZERO (out, Image, Nvalues); + + for (i = 0; i < Nvalues; i++) { + // this is only safe because the initial 120 bytes in Coords match CoordsDisk + memcpy (&out[i].coords, &in[i].coords, sizeof(CoordsDisk)); + out[i].coords.mosaic = NULL; + out[i].coords.offsetMap = NULL; + + // RAW_IMAGE_NAME_LEN == DVO_IMAGE_NAME_LEN + strncpy (out[i].name, in[i].name, RAW_IMAGE_NAME_LEN - 1); + out[i].name[RAW_IMAGE_NAME_LEN - 1] = 0; // force termination + + out[i].tzero = in[i].tzero; + out[i].nstar = in[i].nstar; + out[i].secz = in[i].secz; + out[i].NX = in[i].NX; + out[i].NY = in[i].NY; + out[i].apmifit = in[i].apmifit; + out[i].dapmifit = in[i].dapmifit; + out[i].McalPSF = in[i].McalPSF; + out[i].McalAPER = in[i].McalAPER; + out[i].dMcal = in[i].dMcal; + out[i].McalChiSq = in[i].McalChiSq; + out[i].photcode = in[i].photcode; + out[i].exptime = in[i].exptime; + out[i].sidtime = in[i].sidtime; + out[i].latitude = in[i].latitude; + + out[i].RAo = in[i].RAo; + out[i].DECo = in[i].DECo; + out[i].Radius = in[i].Radius; + out[i].refColorBlue = in[i].refColorBlue; + out[i].refColorRed = in[i].refColorRed; + + out[i].detection_limit = in[i].detection_limit; + out[i].saturation_limit = in[i].saturation_limit; + out[i].cerror = in[i].cerror; + out[i].fwhm_x = in[i].fwhm_x; + out[i].fwhm_y = in[i].fwhm_y; + out[i].trate = in[i].trate; + out[i].ccdnum = in[i].ccdnum; + out[i].flags = in[i].flags; + out[i].imageID = in[i].imageID; + out[i].parentID = in[i].parentID; + out[i].externID = in[i].externID; + out[i].sourceID = in[i].sourceID; + + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].ubercalDist = in[i].ubercalDist; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; + } + return (out); +} + +Image_PS1_V6 *ImageInternalTo_PS1_V6 (Image *in, off_t Nvalues) { + + off_t i; + Image_PS1_V6 *out; + + ALLOCATE_ZERO (out, Image_PS1_V6, Nvalues); + + for (i = 0; i < Nvalues; i++) { + // this is only save because the initial 120 bytes in Coords match CoordsDisk + memcpy (&out[i].coords, &in[i].coords, sizeof(CoordsDisk)); + + // RAW_IMAGE_NAME_LEN == DVO_IMAGE_NAME_LEN + strncpy (out[i].name, in[i].name, RAW_IMAGE_NAME_LEN - 1); + out[i].name[RAW_IMAGE_NAME_LEN - 1] = 0; // force termination + + out[i].tzero = in[i].tzero; + out[i].nstar = in[i].nstar; + out[i].secz = in[i].secz; + out[i].NX = in[i].NX; + out[i].NY = in[i].NY; + out[i].apmifit = in[i].apmifit; + out[i].dapmifit = in[i].dapmifit; + out[i].McalPSF = in[i].McalPSF; + out[i].McalAPER = in[i].McalAPER; + out[i].dMcal = in[i].dMcal; + out[i].McalChiSq = in[i].McalChiSq; + out[i].photcode = in[i].photcode; + out[i].exptime = in[i].exptime; + out[i].sidtime = in[i].sidtime; + out[i].latitude = in[i].latitude; + + out[i].RAo = in[i].RAo; + out[i].DECo = in[i].DECo; + out[i].Radius = in[i].Radius; + out[i].refColorBlue = in[i].refColorBlue; + out[i].refColorRed = in[i].refColorRed; + + out[i].detection_limit = in[i].detection_limit; + out[i].saturation_limit = in[i].saturation_limit; + out[i].cerror = in[i].cerror; + out[i].fwhm_x = in[i].fwhm_x; + out[i].fwhm_y = in[i].fwhm_y; + out[i].trate = in[i].trate; + out[i].ccdnum = in[i].ccdnum; + out[i].flags = in[i].flags; + out[i].imageID = in[i].imageID; + out[i].parentID = in[i].parentID; + out[i].externID = in[i].externID; + out[i].sourceID = in[i].sourceID; + + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].ubercalDist = in[i].ubercalDist; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; + } + return (out); +} + +PhotCode *PhotCode_PS1_V6_To_Internal (PhotCode_PS1_V6 *in, off_t Nvalues) { + + off_t i; + PhotCode *out; + + ALLOCATE_ZERO (out, PhotCode, Nvalues); + + for (i = 0; i < Nvalues; i++) { + strncpy (out[i].name, in[i].name, 31); // out[32], in[32] + out[i].name[31] = 0; // force termination + + out[i].code = in[i].code; + out[i].type = in[i].type; + out[i].C = in[i].C; + out[i].dC = in[i].dC; + out[i].dX = in[i].dX; + out[i].K = in[i].K; + out[i].c1 = in[i].c1; + out[i].c2 = in[i].c2; + out[i].equiv = in[i].equiv; + out[i].Nc = in[i].Nc; + memcpy (out[i].X, in[i].X, 4*sizeof(float)); + + out[i].astromErrSys = in[i].astromErrSys; + out[i].astromErrScale = in[i].astromErrScale; + out[i].astromErrMagScale = in[i].astromErrMagScale; + out[i].photomErrSys = in[i].photomErrSys; + + out[i].photomPoorMask = in[i].photomPoorMask; + out[i].photomBadMask = in[i].photomBadMask; + out[i].astromPoorMask = in[i].astromPoorMask; + out[i].astromBadMask = in[i].astromBadMask; + } + return (out); +} + +PhotCode_PS1_V6 *PhotCode_Internal_To_PS1_V6 (PhotCode *in, off_t Nvalues) { + + off_t i; + PhotCode_PS1_V6 *out; + + ALLOCATE_ZERO (out, PhotCode_PS1_V6, Nvalues); + + for (i = 0; i < Nvalues; i++) { + strncpy (out[i].name, in[i].name, 31); // out[32], in[32] + out[i].name[31] = 0; // force termination + + out[i].code = in[i].code; + out[i].type = in[i].type; + out[i].C = in[i].C; + out[i].dC = in[i].dC; + out[i].dX = in[i].dX; + out[i].K = in[i].K; + out[i].c1 = in[i].c1; + out[i].c2 = in[i].c2; + out[i].equiv = in[i].equiv; + out[i].Nc = in[i].Nc; + memcpy (out[i].X, in[i].X, 4*sizeof(float)); + + out[i].astromErrSys = in[i].astromErrSys; + out[i].astromErrScale = in[i].astromErrScale; + out[i].astromErrMagScale = in[i].astromErrMagScale; + out[i].photomErrSys = in[i].photomErrSys; + + out[i].photomPoorMask = in[i].photomPoorMask; + out[i].photomBadMask = in[i].photomBadMask; + out[i].astromPoorMask = in[i].astromPoorMask; + out[i].astromBadMask = in[i].astromBadMask; + } + return (out); +} Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_elixir.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_elixir.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_elixir.c (revision 40477) @@ -24,5 +24,6 @@ out[i].dM = (in[i].dM == NAN_U_CHAR) ? NAN : in[i].dM * 0.001; out[i].dt = (in[i].dt == NAN_S_SHORT) ? NAN : in[i].dt * 0.001; - out[i].Mcal = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; + out[i].McalPSF = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; + out[i].McalAPER = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; out[i].Map = (in[i].Mgal == NAN_S_SHORT) ? NAN : in[i].Mgal * 0.001; out[i].airmass = (in[i].airmass == NAN_S_SHORT) ? NAN : in[i].airmass* 0.001; @@ -72,5 +73,5 @@ out[i].dM = isnan(in[i].dM ) ? NAN_U_CHAR : in[i].dM * 1000.0; out[i].dt = isnan(in[i].dt ) ? NAN_S_SHORT : in[i].dt * 1000.0; - out[i].Mcal = isnan(in[i].Mcal ) ? NAN_S_SHORT : in[i].Mcal * 1000.0; + out[i].Mcal = isnan(in[i].McalPSF) ? NAN_S_SHORT : in[i].McalPSF * 1000.0; out[i].Mgal = isnan(in[i].Map ) ? NAN_S_SHORT : in[i].Map * 1000.0; out[i].airmass = isnan(in[i].airmass) ? NAN_S_SHORT : in[i].airmass * 1000.0; @@ -109,6 +110,6 @@ // changed for PANSTARRS_DEV_0 (moved from Average to Measure) - primary[0][i].M = (in[i].M == NAN_S_SHORT) ? NAN : in[i].M * 0.001; - primary[0][i].dM = (in[i].dM == NAN_S_SHORT) ? NAN : in[i].dM * 0.001; + primary[0][i].MpsfChp = (in[i].M == NAN_S_SHORT) ? NAN : in[i].M * 0.001; + primary[0][i].dMpsfChp = (in[i].dM == NAN_S_SHORT) ? NAN : in[i].dM * 0.001; primary[0][i].Mchisq= pow (10.0, 0.01*in[i].Xm); @@ -142,6 +143,6 @@ // changed for PANSTARRS_DEV_0 (moved from Average to Measure) - out[i].M = isnan(primary[i].M) ? NAN_S_SHORT : primary[i].M * 1000.0; - out[i].dM = isnan(primary[i].dM) ? NAN_S_SHORT : primary[i].dM * 1000.0; + out[i].M = isnan(primary[i].MpsfChp) ? NAN_S_SHORT : primary[i].MpsfChp * 1000.0; + out[i].dM = isnan(primary[i].dMpsfChp) ? NAN_S_SHORT : primary[i].dMpsfChp * 1000.0; out[i].Xm = 100.0*log10(primary[i].Mchisq); @@ -172,6 +173,6 @@ // added or changed for PANSTARRS_DEV_0 - out[i].M = (in[i].M == NAN_S_SHORT) ? NAN : in[i].M * 0.001; - out[i].dM = (in[i].dM == NAN_S_SHORT) ? NAN : in[i].dM * 0.001; + out[i].MpsfChp = (in[i].M == NAN_S_SHORT) ? NAN : in[i].M * 0.001; + out[i].dMpsfChp = (in[i].dM == NAN_S_SHORT) ? NAN : in[i].dM * 0.001; } return (out); @@ -190,6 +191,6 @@ // added or changed for PANSTARRS_DEV_0 - out[i].M = isnan(in[i].M) ? NAN_S_SHORT : in[i].M * 1000.0; - out[i].dM = isnan(in[i].dM) ? NAN_S_SHORT : in[i].dM * 1000.0; + out[i].M = isnan(in[i].MpsfChp) ? NAN_S_SHORT : in[i].MpsfChp * 1000.0; + out[i].dM = isnan(in[i].dMpsfChp) ? NAN_S_SHORT : in[i].dMpsfChp * 1000.0; } return (out); @@ -223,5 +224,4 @@ out[i].NX = in[i].NX; out[i].NY = in[i].NY; - out[i].Xm = in[i].Xm; out[i].photcode = in[i].source; out[i].exptime = in[i].exptime; @@ -252,6 +252,10 @@ out[i].apmifit = (in[i].apmifit == NAN_S_SHORT) ? NAN : in[i].apmifit * 0.001; out[i].dapmifit = (in[i].dapmifit == NAN_S_SHORT) ? NAN : in[i].dapmifit * 0.001; - out[i].Mcal = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; + + out[i].McalPSF = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; + out[i].McalAPER = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; out[i].dMcal = (in[i].dMcal == NAN_S_SHORT) ? NAN : in[i].dMcal * 0.001; + out[i].McalChiSq = (in[i].dMcal == NAN_S_SHORT) ? NAN : pow(10.0, 0.01*in[i].Xm); + out[i].sidtime = NAN; out[i].latitude = NAN; @@ -295,5 +299,4 @@ out[i].NY = in[i].NY; - out[i].Xm = in[i].Xm; out[i].source = in[i].photcode; out[i].exptime = in[i].exptime; @@ -324,6 +327,8 @@ out[i].apmifit = isnan(in[i].apmifit ) ? NAN_S_SHORT : in[i].apmifit * 1000.0; out[i].dapmifit = isnan(in[i].dapmifit) ? NAN_S_SHORT : in[i].dapmifit * 1000.0; - out[i].Mcal = isnan(in[i].Mcal ) ? NAN_S_SHORT : in[i].Mcal * 1000.0; + + out[i].Mcal = isnan(in[i].McalPSF ) ? NAN_S_SHORT : in[i].McalPSF * 1000.0; out[i].dMcal = isnan(in[i].dMcal ) ? NAN_S_SHORT : in[i].dMcal * 1000.0; + out[i].Xm = isnan(in[i].dMcal ) ? NAN_S_SHORT : 100.0*log10(in[i].McalChiSq); // changed or added for PS1_V1 Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_loneos.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_loneos.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_loneos.c (revision 40477) @@ -26,5 +26,6 @@ out[i].M = (in[i].M == NAN_S_SHORT) ? NAN : in[i].M * 0.001; out[i].dM = (in[i].dM == NAN_U_CHAR) ? NAN : in[i].dM * 0.001; - out[i].Mcal = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; + out[i].McalPSF = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; + out[i].McalAPER = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; out[i].Map = (in[i].M == NAN_S_SHORT) ? NAN : in[i].M * 0.001; out[i].photcode = in[i].source; @@ -59,5 +60,5 @@ out[i].M = isnan(in[i].M ) ? NAN_S_SHORT : in[i].M * 1000.0; out[i].dM = isnan(in[i].dM ) ? NAN_U_CHAR : in[i].dM * 1000.0; - out[i].Mcal = isnan(in[i].Mcal ) ? NAN_S_SHORT : in[i].Mcal * 1000.0; + out[i].Mcal = isnan(in[i].McalPSF) ? NAN_S_SHORT : in[i].McalPSF * 1000.0; out[i].source = in[i].photcode; out[i].t = in[i].t; @@ -89,6 +90,6 @@ // changed for PANSTARRS_DEV_0 (moved from Average to Measure) - primary[0][i].M = (in[i].M == NAN_S_SHORT) ? NAN : in[i].M * 0.001; - primary[0][i].Mchisq= pow (10.0, 0.01*in[i].Xm); + primary[0][i].MpsfChp = (in[i].M == NAN_S_SHORT) ? NAN : in[i].M * 0.001; + primary[0][i].Mchisq = pow (10.0, 0.01*in[i].Xm); // added for PANSTARRS_DEV_0 @@ -123,5 +124,5 @@ // changed for PANSTARRS_DEV_0 (moved from Average to Measure) - out[i].M = isnan(primary[i].M) ? NAN_S_SHORT : primary[i].M * 1000.0; + out[i].M = isnan(primary[i].MpsfChp) ? NAN_S_SHORT : primary[i].MpsfChp * 1000.0; out[i].Xm = 100.0*log10(primary[i].Mchisq); @@ -152,5 +153,5 @@ // added or changed for PANSTARRS_DEV_0 - out[i].M = (in[i].M == NAN_S_SHORT) ? NAN : in[i].M * 0.001; + out[i].MpsfChp = (in[i].M == NAN_S_SHORT) ? NAN : in[i].M * 0.001; } return (out); @@ -169,5 +170,5 @@ // added or changed for PANSTARRS_DEV_0 - out[i].M = isnan(in[i].M) ? NAN_S_SHORT : in[i].M * 1000.0; + out[i].M = isnan(in[i].MpsfChp) ? NAN_S_SHORT : in[i].MpsfChp * 1000.0; } return (out); @@ -202,5 +203,4 @@ out[i].NY = in[i].NY; - out[i].Xm = in[i].Xm; out[i].photcode = in[i].source; out[i].exptime = in[i].exptime; @@ -231,6 +231,10 @@ out[i].apmifit = (in[i].apmifit == NAN_S_SHORT) ? NAN : in[i].apmifit * 0.001; out[i].dapmifit = (in[i].dapmifit == NAN_S_SHORT) ? NAN : in[i].dapmifit * 0.001; - out[i].Mcal = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; + + out[i].McalPSF = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; + out[i].McalAPER = (in[i].Mcal == NAN_S_SHORT) ? NAN : in[i].Mcal * 0.001; out[i].dMcal = (in[i].dMcal == NAN_S_SHORT) ? NAN : in[i].dMcal * 0.001; + out[i].McalChiSq = (in[i].dMcal == NAN_S_SHORT) ? NAN : pow(10.0, 0.01*in[i].Xm); + out[i].sidtime = NAN; out[i].latitude = NAN; @@ -274,5 +278,4 @@ out[i].NY = in[i].NY; - out[i].Xm = in[i].Xm; out[i].source = in[i].photcode; out[i].exptime = in[i].exptime; @@ -303,6 +306,8 @@ out[i].apmifit = isnan(in[i].apmifit ) ? NAN_S_SHORT : in[i].apmifit * 1000.0; out[i].dapmifit = isnan(in[i].dapmifit) ? NAN_S_SHORT : in[i].dapmifit * 1000.0; - out[i].Mcal = isnan(in[i].Mcal ) ? NAN_S_SHORT : in[i].Mcal * 1000.0; + + out[i].Mcal = isnan(in[i].McalPSF ) ? NAN_S_SHORT : in[i].McalPSF * 1000.0; out[i].dMcal = isnan(in[i].dMcal ) ? NAN_S_SHORT : in[i].dMcal * 1000.0; + out[i].Xm = isnan(in[i].dMcal ) ? NAN_S_SHORT : 100.0*log10(in[i].McalChiSq); // changed or added for PS1_V1 Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c (revision 40477) @@ -17,5 +17,6 @@ out[i].M = in[i].M; out[i].dM = in[i].dM; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].airmass = in[i].airmass; @@ -68,5 +69,5 @@ out[i].M = in[i].M; out[i].dM = in[i].dM; - out[i].Mcal = in[i].Mcal; + out[i].Mcal = in[i].McalPSF; out[i].airmass = in[i].airmass; out[i].az = in[i].az; @@ -188,6 +189,7 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].MpsfChp = in[i].M; + out[i].dMpsfChp = in[i].dM; + out[i].Mchisq= pow (10.0, 0.01*in[i].Xm); out[i].Ncode = in[i].Ncode; @@ -206,6 +208,7 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; + out[i].Xm = 100.0*log10(in[i].Mchisq); out[i].Ncode = in[i].Ncode; @@ -245,7 +248,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -316,7 +322,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c (revision 40477) @@ -17,5 +17,6 @@ out[i].M = in[i].M; out[i].dM = in[i].dM; - out[i].Mcal = in[i].Mcal; + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].airmass = in[i].airmass; @@ -68,5 +69,5 @@ out[i].M = in[i].M; out[i].dM = in[i].dM; - out[i].Mcal = in[i].Mcal; + out[i].Mcal = in[i].McalPSF; out[i].airmass = in[i].airmass; out[i].az = in[i].az; @@ -188,6 +189,7 @@ dvo_secfilt_init (&out[i], SECFILT_RESET_ALL); - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].MpsfChp = in[i].M; + out[i].dMpsfChp = in[i].dM; + out[i].Mchisq= pow (10.0, 0.01*in[i].Xm); out[i].Ncode = in[i].Ncode; @@ -206,6 +208,7 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].dM = in[i].dM; + out[i].M = in[i].MpsfChp; + out[i].dM = in[i].dMpsfChp; + out[i].Xm = 100.0*log10(in[i].Mchisq); out[i].Ncode = in[i].Ncode; @@ -245,7 +248,10 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].McalPSF = in[i].Mcal; + out[i].McalAPER = in[i].Mcal; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].McalChiSq = pow(10.0, 0.01*in[i].Xm); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; @@ -316,7 +322,9 @@ out[i].apmifit = in[i].apmifit; out[i].dapmifit = in[i].dapmifit; - out[i].Mcal = in[i].Mcal; + + out[i].Mcal = in[i].McalPSF; out[i].dMcal = in[i].dMcal; - out[i].Xm = in[i].Xm; + out[i].Xm = 100.0*log10(in[i].McalChiSq); + out[i].photcode = in[i].photcode; out[i].exptime = in[i].exptime; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_image.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_image.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_image.c (revision 40477) @@ -223,4 +223,5 @@ if (db[0].format == DVO_FORMAT_PS1_V4) gfits_modify (&db[0].header, "FORMAT", "%s", 1, "PS1_V4"); if (db[0].format == DVO_FORMAT_PS1_V5) gfits_modify (&db[0].header, "FORMAT", "%s", 1, "PS1_V5"); + if (db[0].format == DVO_FORMAT_PS1_V6) gfits_modify (&db[0].header, "FORMAT", "%s", 1, "PS1_V6"); if (db[0].format == DVO_FORMAT_PS1_V5_LOAD) gfits_modify (&db[0].header, "FORMAT", "%s", 1, "PS1_V5_LOAD"); if (db[0].format == DVO_FORMAT_PS1_REF) gfits_modify (&db[0].header, "FORMAT", "%s", 1, "PS1_REF"); @@ -256,5 +257,6 @@ int start_size = header->datasize; - long A = time(NULL); + int PID = getpid(); + long A = PID + time(NULL); srand48(A); @@ -321,7 +323,9 @@ gfits_define_bintable_column (header, "E", "APMIFIT", "aperture correction", "mag", 1.0, 0.0); gfits_define_bintable_column (header, "E", "DAPMIFIT", "apmifit error", "mag", 1.0, 0.0); - gfits_define_bintable_column (header, "E", "MCAL", "calibration mag", "mag", 1.0, 0.0); + gfits_define_bintable_column (header, "E", "MCAL_PSF", "calibration mag for psfs", "mag", 1.0, 0.0); + gfits_define_bintable_column (header, "E", "MCAL_APER", "calibration mag for aper", "mag", 1.0, 0.0); gfits_define_bintable_column (header, "E", "DMCAL", "error on Mcal", "mag", 1.0, 0.0); - gfits_define_bintable_column (header, "I", "XM", "image chisq", "10*log(value)", 1.0, 0.0); + gfits_define_bintable_column (header, "E", "XM", "image chisq", "10*log(value)", 1.0, 0.0); + gfits_define_bintable_column (header, "I", "PADDING", "filler for 8-byte boundaries,", "", 1.0, 0.0); gfits_define_bintable_column (header, "I", "PHOTCODE", "identifier for CCD,", "", 1.0, 0.0); gfits_define_bintable_column (header, "E", "EXPTIME", "exposure time", "seconds", 1.0, 0.0); Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_image_raw.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_image_raw.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_image_raw.c (revision 40477) @@ -60,4 +60,5 @@ if (db[0].format == DVO_FORMAT_PS1_V4) ImageSize = sizeof(Image_PS1_V4); if (db[0].format == DVO_FORMAT_PS1_V5) ImageSize = sizeof(Image_PS1_V5); + if (db[0].format == DVO_FORMAT_PS1_V6) ImageSize = sizeof(Image_PS1_V6); if (db[0].format == DVO_FORMAT_PS1_V5_LOAD) ImageSize = sizeof(Image_PS1_V5_LOAD); if (db[0].format == DVO_FORMAT_PS1_REF) ImageSize = sizeof(Image_PS1_REF); @@ -93,4 +94,5 @@ if (db[0].format == DVO_FORMAT_PS1_V4) gfits_table_mkheader_Image_PS1_V4 (&db[0].theader); if (db[0].format == DVO_FORMAT_PS1_V5) gfits_table_mkheader_Image_PS1_V5 (&db[0].theader); + if (db[0].format == DVO_FORMAT_PS1_V6) gfits_table_mkheader_Image_PS1_V6 (&db[0].theader); if (db[0].format == DVO_FORMAT_PS1_V5_LOAD) gfits_table_mkheader_Image_PS1_V5_LOAD (&db[0].theader); if (db[0].format == DVO_FORMAT_PS1_REF) gfits_table_mkheader_Image_PS1_REF (&db[0].theader); Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_photcode_ops.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_photcode_ops.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_photcode_ops.c (revision 40477) @@ -381,13 +381,17 @@ float Mraw = NAN; + float Mcal = NAN; switch (class) { case MAG_CLASS_PSF: Mraw = measure[0].M; + Mcal = measure[0].McalPSF; break; case MAG_CLASS_KRON: Mraw = measure[0].Mkron; + Mcal = measure[0].McalAPER; break; case MAG_CLASS_APER: Mraw = measure[0].Map; + Mcal = measure[0].McalAPER; break; default: @@ -398,5 +402,5 @@ } float Mflat = isfinite(measure[0].Mflat) ? measure[0].Mflat : 0.0; - float Mcat = Mraw - ZERO_POINT + code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - measure[0].Mcal - Mflat; + float Mcat = Mraw - ZERO_POINT + code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - Mcal - Mflat; /* for DEP, color must be made of PRI/SEC */ @@ -479,11 +483,11 @@ switch (class) { case MAG_CLASS_PSF: - Mave = secfilt[Ns].M; + Mave = secfilt[Ns].MpsfChp; break; case MAG_CLASS_KRON: - Mave = secfilt[Ns].Mkron; + Mave = secfilt[Ns].MkronChp; break; case MAG_CLASS_APER: - Mave = secfilt[Ns].Map; + Mave = secfilt[Ns].MapChp; break; default: @@ -592,11 +596,11 @@ switch (class) { case MAG_CLASS_PSF: - dMave = secfilt[Ns].dM; + dMave = secfilt[Ns].dMpsfChp; break; case MAG_CLASS_KRON: - dMave = secfilt[Ns].dMkron; + dMave = secfilt[Ns].dMkronChp; break; case MAG_CLASS_APER: - dMave = secfilt[Ns].dMap; + dMave = secfilt[Ns].dMapChp; break; default: @@ -677,6 +681,6 @@ Ns2 = photcodes[0].hashNsec[code[0].c2]; - m1 = (Ns1 == -1) ? NAN : secfilt[Ns1].M; - m2 = (Ns2 == -1) ? NAN : secfilt[Ns2].M; + m1 = (Ns1 == -1) ? NAN : secfilt[Ns1].MpsfChp; + m2 = (Ns2 == -1) ? NAN : secfilt[Ns2].MpsfChp; mc = (isnan(m1) || isnan(m2)) ? NAN : (m1 - m2); return (mc); @@ -694,5 +698,5 @@ } else { Ns = photcodes[0].hashNsec[color[0].code]; - m1 = (Ns == -1) ? NAN : secfilt[Ns].M; + m1 = (Ns == -1) ? NAN : secfilt[Ns].MpsfChp; } @@ -708,5 +712,5 @@ } else { Ns = photcodes[0].hashNsec[color[0].code]; - m2 = (Ns == -1) ? NAN : secfilt[Ns].M; + m2 = (Ns == -1) ? NAN : secfilt[Ns].MpsfChp; } mc = (isnan(m1) || isnan(m2)) ? NAN : (m1 - m2); @@ -727,11 +731,11 @@ switch (class) { case MAG_CLASS_PSF: - Mstdev = secfilt[Ns].Mstdev; + Mstdev = secfilt[Ns].sMpsfChp; break; case MAG_CLASS_KRON: - Mstdev = secfilt[Ns].sMkron; + Mstdev = secfilt[Ns].sMkronChp; break; case MAG_CLASS_APER: - Mstdev = secfilt[Ns].sMap; + Mstdev = secfilt[Ns].sMapChp; break; default: @@ -1040,5 +1044,5 @@ } else { Ns = photcodes[0].hashNsec[code[0].code]; - M1 = (Ns == -1) ? NAN : secfilt[Ns].M; + M1 = (Ns == -1) ? NAN : secfilt[Ns].MpsfChp; } @@ -1056,5 +1060,5 @@ } else { Ns = photcodes[0].hashNsec[code[0].code]; - M2 = (Ns == -1) ? NAN : secfilt[Ns].M; + M2 = (Ns == -1) ? NAN : secfilt[Ns].MpsfChp; } @@ -1130,7 +1134,7 @@ // measure.M has the static ZERO_POINT (25.0) applied, but not measure.Flux - float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C; - float Moff = Mcal - ZERO_POINT + 8.9; - float Foff = 3630.8 * MagToFlux(Mcal); + float Mzpt = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C; + float Moff = Mzpt - ZERO_POINT + 8.9; + float Foff = 3630.8 * MagToFlux(Mzpt); float Fcat = NAN; switch (class) { @@ -1158,7 +1162,7 @@ // measure.M has the static ZERO_POINT (25.0) applied, but not measure.Flux - float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C; - float Moff = Mcal - ZERO_POINT + 8.9; - float Foff = 3630.8 * MagToFlux(Mcal); + float Mzpt = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C; + float Moff = Mzpt - ZERO_POINT + 8.9; + float Foff = 3630.8 * MagToFlux(Mzpt); float Fcat = NAN; switch (class) { @@ -1203,8 +1207,12 @@ // measure.M has the static ZERO_POINT (25.0) applied, but not measure.Flux + + // use Mcal APER for aperture-like data + float Mcal = (class == MAG_CLASS_PSF) ? measure[0].McalPSF : measure[0].McalAPER; float Mflat = isfinite(measure[0].Mflat) ? measure[0].Mflat : 0.0; - float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - measure[0].Mcal - Mflat; - float Moff = Mcal - ZERO_POINT + 8.9; - float Foff = 3630.8 * MagToFlux(Mcal); + + float Mzpt = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - Mcal - Mflat; + float Moff = Mzpt - ZERO_POINT + 8.9; + float Foff = 3630.8 * MagToFlux(Mzpt); float Fcat = NAN; switch (class) { @@ -1289,11 +1297,11 @@ switch (class) { case MAG_CLASS_PSF: - Fave = MagToFlux(secfilt[Ns].M - 8.9); + Fave = MagToFlux(secfilt[Ns].MpsfChp - 8.9); break; case MAG_CLASS_KRON: - Fave = MagToFlux(secfilt[Ns].Mkron - 8.9); + Fave = MagToFlux(secfilt[Ns].MkronChp - 8.9); break; case MAG_CLASS_APER: - Fave = MagToFlux(secfilt[Ns].Map - 8.9); + Fave = MagToFlux(secfilt[Ns].MapChp - 8.9); break; default: @@ -1350,11 +1358,11 @@ switch (class) { case MAG_CLASS_PSF: - dFave = secfilt[Ns].dM * MagToFlux(secfilt[Ns].M - 8.9); + dFave = secfilt[Ns].dMpsfChp * MagToFlux(secfilt[Ns].MpsfChp - 8.9); break; case MAG_CLASS_KRON: - dFave = secfilt[Ns].dMkron * MagToFlux(secfilt[Ns].Mkron - 8.9); + dFave = secfilt[Ns].dMkronChp * MagToFlux(secfilt[Ns].MkronChp - 8.9); break; case MAG_CLASS_APER: - dFave = secfilt[Ns].dMap * MagToFlux(secfilt[Ns].Map - 8.9); + dFave = secfilt[Ns].dMapChp * MagToFlux(secfilt[Ns].MapChp - 8.9); break; default: @@ -1470,7 +1478,7 @@ // measure.M has the static ZERO_POINT (25.0) applied, but not measure.Flux - float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C; - float Moff = Mcal - ZERO_POINT + 8.9; - float Foff = 3630.8 * MagToFlux(Mcal); + float Mzpt = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C; + float Moff = Mzpt - ZERO_POINT + 8.9; + float Foff = 3630.8 * MagToFlux(Mzpt); // use dFlux if we can, but use dMag if we must: @@ -1516,7 +1524,7 @@ // measure.M has the static ZERO_POINT (25.0) applied, but not measure.Flux - float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C; - float Moff = Mcal - ZERO_POINT + 8.9; - float Foff = 3630.8 * MagToFlux(Mcal); + float Mzpt = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C; + float Moff = Mzpt - ZERO_POINT + 8.9; + float Foff = 3630.8 * MagToFlux(Mzpt); // use dFlux if we can, but use dMag if we must: @@ -1579,8 +1587,11 @@ // measure.M has the static ZERO_POINT (25.0) applied, but not measure.Flux + // XXX fix this too: float Mflat = isfinite(measure[0].Mflat) ? measure[0].Mflat : 0.0; - float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - measure[0].Mcal - Mflat; - float Moff = Mcal - ZERO_POINT + 8.9; - float Foff = 3630.8 * MagToFlux(Mcal); + float Mcal = (class == MAG_CLASS_PSF) ? measure[0].McalPSF : measure[0].McalAPER; + + float Mzpt = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - Mcal - Mflat; + float Moff = Mzpt - ZERO_POINT + 8.9; + float Foff = 3630.8 * MagToFlux(Mzpt); // use dFlux if we can, but use dMag if we must: @@ -1649,8 +1660,8 @@ break; case MAG_CLASS_KRON: - // Mraw = measure[0].Mkron; + Mraw = measure[0].Mkron; break; case MAG_CLASS_APER: - // Mraw = measure[0].Map; + // Mraw = measure[0].Map; // MeasureTiny does not have Map break; default: @@ -1677,8 +1688,8 @@ break; case MAG_CLASS_KRON: - // Mraw = measure[0].Mkron; + Mraw = measure[0].Mkron; break; case MAG_CLASS_APER: - // Mraw = measure[0].Map; + // Mraw = measure[0].Map; // MeasureTiny does not have Map break; default: @@ -1706,8 +1717,8 @@ break; case MAG_CLASS_KRON: - // Mraw = measure[0].Mkron; + Mraw = measure[0].Mkron; break; case MAG_CLASS_APER: - // Mraw = measure[0].Map; + // Mraw = measure[0].Map; // MeasureTiny does not have Map break; default: @@ -1742,13 +1753,17 @@ float Mraw = NAN; + float Mcal = NAN; switch (class) { case MAG_CLASS_PSF: Mraw = measure[0].M; + Mcal = measure[0].McalPSF; break; case MAG_CLASS_KRON: - // Mraw = measure[0].Mkron; + Mraw = measure[0].Mkron; + Mcal = measure[0].McalAPER; break; case MAG_CLASS_APER: - // Mraw = measure[0].Map; + // Mraw = measure[0].Map; // MeasureTiny does not have Map + // Mcal = measure[0].McalAPER; break; default: @@ -1759,5 +1774,5 @@ } float Mflat = isfinite(measure[0].Mflat) ? measure[0].Mflat : 0.0; - float Mcat = Mraw - ZERO_POINT + code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - measure[0].Mcal - Mflat; + float Mcat = Mraw - ZERO_POINT + code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - Mcal - Mflat; /* for DEP, color must be made of PRI/SEC */ @@ -1799,8 +1814,8 @@ break; case MAG_CLASS_KRON: - // Mraw = thisone[0].Mkron; + Mraw = thisone[0].Mkron; break; case MAG_CLASS_APER: - // Mraw = thisone[0].Map; + // Mraw = thisone[0].Map; // MeasureTiny does not have Map break; default: @@ -1840,11 +1855,11 @@ switch (class) { case MAG_CLASS_PSF: - Mave = secfilt[Ns].M; + Mave = secfilt[Ns].MpsfChp; break; case MAG_CLASS_KRON: - Mave = secfilt[Ns].Mkron; + Mave = secfilt[Ns].MkronChp; break; case MAG_CLASS_APER: - Mave = secfilt[Ns].Map; + Mave = secfilt[Ns].MapChp; break; default: @@ -1927,6 +1942,6 @@ Ns2 = photcodes[0].hashNsec[code[0].c2]; - m1 = (Ns1 == -1) ? NAN : secfilt[Ns1].M; - m2 = (Ns2 == -1) ? NAN : secfilt[Ns2].M; + m1 = (Ns1 == -1) ? NAN : secfilt[Ns1].MpsfChp; + m2 = (Ns2 == -1) ? NAN : secfilt[Ns2].MpsfChp; mc = (isnan(m1) || isnan(m2)) ? NAN : (m1 - m2); return (mc); @@ -1944,5 +1959,5 @@ } else { Ns = photcodes[0].hashNsec[color[0].code]; - m1 = (Ns == -1) ? NAN : secfilt[Ns].M; + m1 = (Ns == -1) ? NAN : secfilt[Ns].MpsfChp; } @@ -1958,5 +1973,5 @@ } else { Ns = photcodes[0].hashNsec[color[0].code]; - m2 = (Ns == -1) ? NAN : secfilt[Ns].M; + m2 = (Ns == -1) ? NAN : secfilt[Ns].MpsfChp; } mc = (isnan(m1) || isnan(m2)) ? NAN : (m1 - m2); @@ -1973,5 +1988,5 @@ Ns = photcodes[0].hashNsec[code[0].code]; - dM = (Ns == -1) ? NAN : secfilt[Ns].dM; + dM = (Ns == -1) ? NAN : secfilt[Ns].dMpsfChp; return (dM); } @@ -2010,5 +2025,5 @@ } else { Ns = photcodes[0].hashNsec[code[0].code]; - M1 = (Ns == -1) ? NAN : secfilt[Ns].M; + M1 = (Ns == -1) ? NAN : secfilt[Ns].MpsfChp; } @@ -2026,5 +2041,5 @@ } else { Ns = photcodes[0].hashNsec[code[0].code]; - M2 = (Ns == -1) ? NAN : secfilt[Ns].M; + M2 = (Ns == -1) ? NAN : secfilt[Ns].MpsfChp; } @@ -2122,5 +2137,5 @@ LENSFIELD(E2) -# if (1) +# if (0) float GalphotValue_GAL_MAG (PhotCode *code, dvoMagClassType class, GalPhot *galphot, int Ngalphot) { int n; @@ -2150,8 +2165,10 @@ # endif -# define GALPHOT_FIELD(NAME, VALUE) \ - float GalphotValue_##NAME (PhotCode *code, dvoMagClassType class, GalPhot *galphot, int Ngalphot) { \ +// the block above is an example of the generic version below + +# define GALPHOT_FIELD(NAME, VALUE, TYPE, DEFAULT) \ + TYPE GalphotValue_##NAME (PhotCode *code, dvoMagClassType class, GalPhot *galphot, int Ngalphot) { \ int n; \ - if (code == NULL) return NAN; \ + if (code == NULL) return DEFAULT; \ for (n = 0; n < Ngalphot; n++) { \ short equivCode = GetPhotcodeEquivCodebyCode (galphot[n].photcode); \ @@ -2169,23 +2186,29 @@ break; \ default: \ - return NAN; \ + return DEFAULT; \ } \ - float value = galphot[n].VALUE; \ + TYPE value = galphot[n].VALUE; \ return (value); \ } \ - return NAN; \ - } - -// GALPHOT_FIELD(GAL_MAG, mag) -GALPHOT_FIELD(GAL_MAG_ERR, magErr) -GALPHOT_FIELD(GAL_MAJ, majorAxis) -GALPHOT_FIELD(GAL_MAJ_ERR, majorAxisErr) -GALPHOT_FIELD(GAL_MIN, minorAxis) -GALPHOT_FIELD(GAL_MIN_ERR, minorAxisErr) -GALPHOT_FIELD(GAL_THETA, theta) -GALPHOT_FIELD(GAL_THETA_ERR, thetaErr) -GALPHOT_FIELD(GAL_INDEX, index) -GALPHOT_FIELD(GAL_CHISQ, chisq) -GALPHOT_FIELD(GAL_NPIX, Npix) -GALPHOT_FIELD(GAL_FLAGS, flags) -GALPHOT_FIELD(GAL_TYPE, modelType) + return DEFAULT; \ + } + +GALPHOT_FIELD(GAL_MAG, mag, float, NAN) +GALPHOT_FIELD(GAL_MAG_ERR, magErr, float, NAN) +GALPHOT_FIELD(GAL_MAJ, majorAxis, float, NAN) +GALPHOT_FIELD(GAL_MAJ_ERR, majorAxisErr, float, NAN) +GALPHOT_FIELD(GAL_MIN, minorAxis, float, NAN) +GALPHOT_FIELD(GAL_MIN_ERR, minorAxisErr, float, NAN) +GALPHOT_FIELD(GAL_THETA, theta, float, NAN) +GALPHOT_FIELD(GAL_THETA_ERR, thetaErr, float, NAN) +GALPHOT_FIELD(GAL_INDEX, index, float, NAN) +GALPHOT_FIELD(GAL_CHISQ, chisq, float, NAN) +GALPHOT_FIELD(GAL_NPIX, Npix, float, NAN) + +GALPHOT_FIELD(GAL_TYPE, modelType, short, 0) + +GALPHOT_FIELD(GAL_FLAGS, flags, unsigned int, 0) +GALPHOT_FIELD(GAL_OBJ_ID, objID, unsigned int, 0) +GALPHOT_FIELD(GAL_CAT_ID, catID, unsigned int, 0) +GALPHOT_FIELD(GAL_DET_ID, detID, unsigned int, 0) +GALPHOT_FIELD(GAL_IMAGE_ID, imageID, unsigned int, 0) Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_tiny_values.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_tiny_values.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvo_tiny_values.c (revision 40477) @@ -19,5 +19,7 @@ measureT[0].D = measure[0].D; measureT[0].M = measure[0].M; - measureT[0].Mcal = measure[0].Mcal; + measureT[0].Mkron = measure[0].Mkron; + measureT[0].McalPSF = measure[0].McalPSF; + measureT[0].McalAPER = measure[0].McalAPER; measureT[0].dM = measure[0].dM; measureT[0].airmass = measure[0].airmass; Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvosorts.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvosorts.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/dvosorts.c (revision 40477) @@ -7,5 +7,6 @@ # define SWAPFUNC(A,B){ Average tmp; tmp = ave[A]; ave[A] = ave[B]; ave[B] = tmp; } -# define COMPARE(A,B)(ave[A].R < ave[B].R) +# define COMPARE(A,B)((!isfinite(ave[A].R) && isfinite(ave[B].R)) || (ave[A].R < ave[B].R)) +// # define COMPARE(A,B) (ave[A].R < ave[B].R) OHANA_SORT (N, COMPARE, SWAPFUNC); @@ -50,5 +51,6 @@ itmp = S[A]; S[A] = S[B]; S[B] = itmp; \ } -# define COMPARE(A,B)(X[A] < X[B]) +// # define COMPARE(A,B) (X[A] < X[B]) +# define COMPARE(A,B)((!isfinite(X[A]) && isfinite(X[B])) || (X[A] < X[B])) OHANA_SORT (N, COMPARE, SWAPFUNC); @@ -81,5 +83,5 @@ itmp = S[A]; S[A] = S[B]; S[B] = itmp; \ } -# define COMPARE(A,B)(X[S[A]] < X[S[B]]) +# define COMPARE(A,B)((!isfinite(X[S[A]]) && isfinite(X[S[B]])) || (X[S[A]] < X[S[B]])) OHANA_SORT (N, COMPARE, SWAPFUNC); @@ -93,5 +95,6 @@ # define SWAPFUNC(A,B){ SkyRegion tmp; tmp = region[A]; region[A] = region[B]; region[B] = tmp; } -# define COMPARE(A,B)(region[A].Dmin < region[B].Dmin) +// # define COMPARE(A,B)(region[A].Dmin < region[B].Dmin) +# define COMPARE(A,B)((!isfinite(region[A].Dmin) && isfinite(region[B].Dmin)) || (region[A].Dmin < region[B].Dmin)) OHANA_SORT (N, COMPARE, SWAPFUNC); Index: /branches/czw_branch/20170908/Ohana/src/libdvo/src/skyregion_ops.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libdvo/src/skyregion_ops.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libdvo/src/skyregion_ops.c (revision 40477) @@ -319,8 +319,14 @@ // a region like -10 10 0 20 will become 350 10 0 20 and return all regions on both sides of 0,360 +// a region like -10 370 should NOT become 350 10, but should instead become 0 360 SkyList *SkyListByBounds (SkyTable *table, int depth, double Rmin, double Rmax, double Dmin, double Dmax) { int i, j, Ns; SkyList *list, *extra; + + if (Rmax - Rmin > 360.0) { + Rmin = 0.0; + Rmax = 360.0; + } Rmin = ohana_normalize_angle (Rmin); Index: /branches/czw_branch/20170908/Ohana/src/libkapa/include/kapa.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libkapa/include/kapa.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libkapa/include/kapa.h (revision 40477) @@ -239,5 +239,5 @@ int KapaSetFont (int fd, char *name, int size); int KapaSendLabel (int fd, char *string, int mode); -int KapaSendTextline (int fd, char *string, float x, float y, float angle); +int KapaSendTextline (int fd, char *string, float x, float y, float angle, int justify, int color); int KapaSetLimits (int fd, Graphdata *graphmode); int KapaGetLimits (int fd, float *dx, float *dy); @@ -292,4 +292,5 @@ void bDrawBufferFree (bDrawBuffer *buffer); void bDrawSetBuffer (bDrawBuffer *buffer); +void bDrawSetColor (bDrawBuffer *buffer, bDrawColor color); void bDrawSetStyle (bDrawBuffer *buffer, bDrawColor color, int lw, int lt); void bDrawPoint (bDrawBuffer *buffer, int x, int y); Index: /branches/czw_branch/20170908/Ohana/src/libkapa/src/DrawRotString.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libkapa/src/DrawRotString.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libkapa/src/DrawRotString.c (revision 40477) @@ -12,4 +12,5 @@ static unsigned long RotBackground; +// fore and back are X colors int DrawRotTextInit (Display *display, Window window, GC gc, unsigned long fore, unsigned long back) { Index: /branches/czw_branch/20170908/Ohana/src/libkapa/src/KapaWindow.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libkapa/src/KapaWindow.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libkapa/src/KapaWindow.c (revision 40477) @@ -94,5 +94,6 @@ graphdata[0].xmax = graphdata[0].ymax = 1.0; - graphdata[0].style = graphdata[0].ptype = 2; + graphdata[0].style = KAPA_PLOT_POINTS; + graphdata[0].ptype = KAPA_POINT_BOX_SOLID; graphdata[0].ltype = graphdata[0].color = 0; graphdata[0].etype = graphdata[0].ebar = 0; @@ -357,8 +358,11 @@ } -int KapaSendTextline (int fd, char *string, float x, float y, float angle) { - +int KapaSendTextline (int fd, char *string, float x, float y, float angle, int justify, int color) { + + // must be in range 0 - 8 + justify = MIN(MAX(justify, 0), 8); + KiiSendCommand (fd, 4, "PTXT"); - KiiSendMessage (fd, "%f %f %f ", x, y, angle); + KiiSendMessage (fd, "%f %f %f %d %d", x, y, angle, justify, color); KiiSendData (fd, string, strlen(string)); KiiWaitAnswer (fd, "DONE"); Index: /branches/czw_branch/20170908/Ohana/src/libkapa/src/PSRotFont.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libkapa/src/PSRotFont.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libkapa/src/PSRotFont.c (revision 40477) @@ -90,24 +90,24 @@ /* check for special characters */ if (!code && !protect) { - /* superscript character (^) */ + /* subscript character (_) */ if (N == 94) { PSDumpRotSegment (f, segment, &Nseg); - SetRotFont (currentname, (int)(0.8*currentsize)); + PSSetFont (f, currentname, (int)(0.8*currentsize)); currentfont = GetRotFontData (¤tscale); Yoff = 0.75*currentscale*dY; fprintf (f, "0 %d rmoveto\n", Yoff); YoffBase += Yoff; - PSSetFont (f, currentname, currentsize); - continue; - } - /* subscript character (_) */ + // PSSetFont (f, currentname, currentsize); + continue; + } + /* superscript character (^) */ if (N == 95) { PSDumpRotSegment (f, segment, &Nseg); - SetRotFont (currentname, (int)(0.8*currentsize)); + PSSetFont (f, currentname, (int)(0.8*currentsize)); currentfont = GetRotFontData (¤tscale); Yoff = -0.5*currentscale*dY; fprintf (f, "0 %d rmoveto\n", Yoff); YoffBase += Yoff; - PSSetFont (f, currentname, currentsize); + // PSSetFont (f, currentname, currentsize); continue; } @@ -115,9 +115,9 @@ if (N == 124) { PSDumpRotSegment (f, segment, &Nseg); - SetRotFont (currentname, basesize); + PSSetFont (f, currentname, basesize); currentfont = GetRotFontData (¤tscale); fprintf (f, "0 %d rmoveto\n", -YoffBase); YoffBase = 0; - PSSetFont (f, currentname, currentsize); + // PSSetFont (f, currentname, currentsize); continue; } @@ -145,22 +145,22 @@ PSDumpRotSegment (f, segment, &Nseg); if (string[i+1] == 'h') { - SetRotFont ("helvetica", currentsize); - currentfont = GetRotFontData (¤tscale); - PSSetFont (f, currentname, currentsize); + PSSetFont (f, "helvetica", currentsize); + currentfont = GetRotFontData (¤tscale); + // PSSetFont (f, currentname, currentsize); } if (string[i+1] == 't') { - SetRotFont ("times", currentsize); - currentfont = GetRotFontData (¤tscale); - PSSetFont (f, currentname, currentsize); + PSSetFont (f, "times", currentsize); + currentfont = GetRotFontData (¤tscale); + // PSSetFont (f, currentname, currentsize); } if (string[i+1] == 'c') { - SetRotFont ("courier", currentsize); - currentfont = GetRotFontData (¤tscale); - PSSetFont (f, currentname, currentsize); + PSSetFont (f, "courier", currentsize); + currentfont = GetRotFontData (¤tscale); + // PSSetFont (f, currentname, currentsize); } if (string[i+1] == 's') { - SetRotFont ("symbol", currentsize); - currentfont = GetRotFontData (¤tscale); - PSSetFont (f, currentname, currentsize); + PSSetFont (f, "symbol", currentsize); + currentfont = GetRotFontData (¤tscale); + // PSSetFont (f, currentname, currentsize); } i++; @@ -176,5 +176,5 @@ fprintf (f, "stroke grestore\n"); free (segment); - SetRotFont (basename, basesize); + PSSetFont (f, basename, basesize); } Index: /branches/czw_branch/20170908/Ohana/src/libkapa/src/bDrawFuncs.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libkapa/src/bDrawFuncs.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libkapa/src/bDrawFuncs.c (revision 40477) @@ -76,9 +76,16 @@ // } -void bDrawSetStyle (bDrawBuffer *buffer, bDrawColor color, int lw, int lt) { +// "bDrawColor color" is one of the hardwired colors in KapaColors.c +void bDrawSetColor (bDrawBuffer *buffer, bDrawColor color) { buffer->bColor = color; buffer->bColor_R = buffer->palette[color].red; buffer->bColor_G = buffer->palette[color].green; buffer->bColor_B = buffer->palette[color].blue; + + return; +} + +void bDrawSetStyle (bDrawBuffer *buffer, bDrawColor color, int lw, int lt) { + bDrawSetColor (buffer, color); buffer->bWeight = lw; @@ -121,4 +128,5 @@ } +// x1,y1 is lower-left corner, x2,y2 is upper-right corner void bDrawRectOpen (bDrawBuffer *buffer, double x1, double y1, double x2, double y2) { Index: /branches/czw_branch/20170908/Ohana/src/libohana/include/ohana.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libohana/include/ohana.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libohana/include/ohana.h (revision 40477) @@ -485,4 +485,5 @@ /* in bisection.c */ int ohana_bisection_double (double *values, int Nvalues, double threshold); +int ohana_bisection_int (int *values, int Nvalues, int threshold); unsigned int sprintf_float (char *output, float value); Index: /branches/czw_branch/20170908/Ohana/src/libohana/include/ohana_sort.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libohana/include/ohana_sort.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libohana/include/ohana_sort.h (revision 40477) @@ -68,4 +68,5 @@ void fsort (float *value, int N); void isort (int *value, int N); +void llsort (long long int *value, int N); void dsortpair (double *X, double *Y, int N); Index: /branches/czw_branch/20170908/Ohana/src/libohana/src/bisection.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libohana/src/bisection.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libohana/src/bisection.c (revision 40477) @@ -32,2 +32,34 @@ return (N); } + +// return the index of the last value < threshold +int ohana_bisection_int (int *values, int Nvalues, int threshold) { + + int Nlo = 0; + int Nhi = Nvalues - 1; + + if (Nvalues < 1) return (-1); + if (values[Nlo] > threshold) return (-1); + + if (Nvalues < 2) return (0); + if (values[Nhi] < threshold) return (Nhi); + + int N; + while (Nhi - Nlo > 4) { + N = 0.5*(Nlo + Nhi); + if (values[N] < threshold) { + Nlo = MAX(N, 0); + } else { + Nhi = MIN(N + 1, Nvalues - 1); + } + } + // values[Nlo] < threshold + // values[Nhi] >= threshold + + for (N = Nlo; N < Nhi; N++) { + if (values[N] >= threshold) { + return (N-1); + } + } + return (N); +} Index: /branches/czw_branch/20170908/Ohana/src/libohana/src/errors.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libohana/src/errors.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libohana/src/errors.c (revision 40477) @@ -1,9 +1,25 @@ # include "ohana.h" -static char errorline[1024]; +int Nline = 0; +int NLINE = 0; +char **errorlines = NULL; int init_error () { - bzero (errorline, 1024); + if (!errorlines) { + NLINE = 10; + ALLOCATE (errorlines, char *, NLINE); + for (int i = 0; i < NLINE; i++) { + errorlines[i] = NULL; + } + } + + for (int i = 0; i < NLINE; i++) { + FREE (errorlines[i]); + errorlines[i] = NULL; + } + + Nline = 0; + return (TRUE); } @@ -11,6 +27,15 @@ int push_error (char *line) { - bzero (errorline, 1024); - strncpy (errorline, line, 1023); + errorlines[Nline] = strcreate (line); + Nline ++; + + if (Nline >= NLINE) { + NLINE += 10; + REALLOCATE (errorlines, char *, NLINE); + for (int i = Nline; i < NLINE; i++) { + errorlines[i] = NULL; + } + } + return (TRUE); } @@ -18,5 +43,10 @@ int print_error () { - gprint (GP_ERR, "%s\n", errorline); + for (int i = 0; i < Nline; i++) { + gprint (GP_ERR, "%s\n", errorlines[i]); + } + + init_error(); + return (TRUE); } Index: /branches/czw_branch/20170908/Ohana/src/libohana/src/isolate_elements.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libohana/src/isolate_elements.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libohana/src/isolate_elements.c (revision 40477) @@ -171,5 +171,8 @@ // order matches convert_to_RPN.c if (!strncmp (c, "?", 1)) return (TRUE); + // if (!strncmp (c, ":", 1)) return (TRUE); + + // do not include : in this list: an unisolated colon acts as a modifier if (!strncmp (c, "^", 1)) return (TRUE); Index: /branches/czw_branch/20170908/Ohana/src/libohana/src/sorts.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/libohana/src/sorts.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/libohana/src/sorts.c (revision 40477) @@ -30,4 +30,16 @@ # define SWAPFUNC(A,B){ int tmp = value[A]; value[A] = value[B]; value[B] = tmp; } +# define COMPARE(A,B)(value[A] < value[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + +void llsort (long long int *value, int N) { + +# define SWAPFUNC(A,B){ long long int tmp = value[A]; value[A] = value[B]; value[B] = tmp; } # define COMPARE(A,B)(value[A] < value[B]) Index: /branches/czw_branch/20170908/Ohana/src/markrock/src/find_slow_rocks.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/markrock/src/find_slow_rocks.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/markrock/src/find_slow_rocks.c (revision 40477) @@ -69,5 +69,5 @@ RD_to_XY (&X1[j], &Y1[j], R1[j], D1[j], &catstats[0].coords); T1[j] = catalog[0].measure[m+j].t; - M1[j] = catalog[0].measure[m+j].M - catalog[0].measure[m+j].Mcal; + M1[j] = catalog[0].measure[m+j].M - catalog[0].measure[m+j].McalPSF; } dt = T1[1] - T1[0]; @@ -86,5 +86,5 @@ RD_to_XY (&X, &Y, R, D, &catstats[0].coords); T = catalog[0].measure[m+j].t; - M = catalog[0].measure[m+j].M - catalog[0].measure[m+j].Mcal; + M = catalog[0].measure[m+j].M - catalog[0].measure[m+j].McalPSF; if (T1[0] == T) continue; if (T1[1] == T) continue; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/Makefile =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/Makefile (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/Makefile (revision 40477) @@ -82,4 +82,5 @@ $(SRC)/wcs.$(ARCH).o \ $(SRC)/imsub.$(ARCH).o \ +$(SRC)/jdtolst.$(ARCH).o \ $(SRC)/imfit.$(ARCH).o \ $(SRC)/imfit-fgauss.$(ARCH).o \ Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/cdot.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/cdot.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/cdot.c (revision 40477) @@ -22,5 +22,5 @@ /* set point style and errorbar mode (these are NOT sticky) */ - graphmode.style = 2; + graphmode.style = KAPA_PLOT_POINTS; /* points */ graphmode.etype = 0; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/cgrid.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/cgrid.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/cgrid.c (revision 40477) @@ -95,4 +95,38 @@ } + int JustifyRA = 5; + double JustifyDEC = 5; + if ((N = get_argument (argc, argv, "-justify-ra"))) { + remove_argument (N, &argc, argv); + JustifyRA = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-justify-dec"))) { + remove_argument (N, &argc, argv); + JustifyDEC = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + double LabelRA = NAN; + double LabelDEC = NAN; + if ((N = get_argument (argc, argv, "-label-ra"))) { + remove_argument (N, &argc, argv); + LabelRA = atof (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-label-dec"))) { + remove_argument (N, &argc, argv); + LabelDEC = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + int LabelColor = KapaColorByName ("black"); + if ((N = get_argument (argc, argv, "-label-color"))) { + remove_argument (N, &argc, argv); + LabelColor = KapaColorByName (argv[N]); + if (LabelColor == -1) return (FALSE); + remove_argument (N, &argc, argv); + } + if (!style_args (&graphmode, &argc, argv, &kapa)) return FALSE; @@ -196,10 +230,12 @@ if (Labels) { char line[16], format[8]; - double xt, yt, dx, dy, frac; - dx = +0.01 * (graphmode.xmax - graphmode.xmin); - dy = -0.02 * (graphmode.ymax - graphmode.ymin); - + double xt, yt, frac; + // dx = +0.01 * (graphmode.xmax - graphmode.xmin); + // dy = -0.02 * (graphmode.ymax - graphmode.ymin); + + if (isnan(LabelRA)) LabelRA = graphmode.coords.crval1; + if (isnan(LabelDEC)) LabelDEC = graphmode.coords.crval2; for (r = firstRA; r <= graphmode.coords.crval1 + 180.0; r += minorRA) { - status = RD_to_XY (&xt, &yt, r, graphmode.coords.crval2, &graphmode.coords); + status = RD_to_XY (&xt, &yt, r, LabelDEC, &graphmode.coords); if (!status) continue; if (xt < graphmode.xmin) continue; @@ -212,14 +248,15 @@ } if (frac <= 0.0) frac = 0.0; - snprintf (format, 8, "%%.%df", (int) frac); if (RAbyHour) { + snprintf (format, 8, "%%.%df^h", (int) frac); snprintf (line, 16, format, r / 15.0); } else { + snprintf (format, 8, "%%.%df^o", (int) frac); snprintf (line, 16, format, r); } - KapaSendTextline (kapa, line, xt + dx, yt + dy, 0.0); + KapaSendTextline (kapa, line, xt, yt, 0.0, JustifyRA, LabelColor); } for (r = firstRA; r >= graphmode.coords.crval1 - 180.0; r -= minorRA) { - status = RD_to_XY (&xt, &yt, r, graphmode.coords.crval2, &graphmode.coords); + status = RD_to_XY (&xt, &yt, r, LabelDEC, &graphmode.coords); if (!status) continue; if (xt < graphmode.xmin) continue; @@ -232,14 +269,15 @@ } if (frac <= 0.0) frac = 0.0; - snprintf (format, 8, "%%.%df", (int) frac); if (RAbyHour) { + snprintf (format, 8, "%%.%df^h", (int) frac); snprintf (line, 16, format, r / 15.0); } else { + snprintf (format, 8, "%%.%df^o", (int) frac); snprintf (line, 16, format, r); } - KapaSendTextline (kapa, line, xt + dx, yt + dy, 0.0); + KapaSendTextline (kapa, line, xt, yt, 0.0, JustifyRA, LabelColor); } for (d = firstDEC; d <= graphmode.coords.crval2 + 90.0; d += minorDEC) { - status = RD_to_XY (&xt, &yt, graphmode.coords.crval1, d, &graphmode.coords); + status = RD_to_XY (&xt, &yt, LabelRA, d, &graphmode.coords); if (!status) continue; if (xt < graphmode.xmin) continue; @@ -252,10 +290,10 @@ } if (frac <= 0.0) frac = 0.0; - snprintf (format, 8, "%%.%df", (int) frac); + snprintf (format, 8, "%%.%df^o", (int) frac); snprintf (line, 16, format, d); - KapaSendTextline (kapa, line, xt + dx, yt + dy, 0.0); + KapaSendTextline (kapa, line, xt, yt, 0.0, JustifyDEC, LabelColor); } for (d = firstDEC; d >= graphmode.coords.crval2 - 90.0; d -= minorDEC) { - status = RD_to_XY (&xt, &yt, graphmode.coords.crval1, d, &graphmode.coords); + status = RD_to_XY (&xt, &yt, LabelRA, d, &graphmode.coords); if (!status) continue; if (xt < graphmode.xmin) continue; @@ -268,7 +306,7 @@ } if (frac <= 0.0) frac = 0.0; - snprintf (format, 8, "%%.%df", (int) frac); + snprintf (format, 8, "%%.%df^o", (int) frac); snprintf (line, 16, format, d); - KapaSendTextline (kapa, line, xt + dx, yt + dy, 0.0); + KapaSendTextline (kapa, line, xt, yt, 0.0, JustifyDEC, LabelColor); } } @@ -276,6 +314,6 @@ /* send the line segments as connect-points */ Xvec.Nelements = Yvec.Nelements = N; - graphmode.style = 2; /* points */ - graphmode.ptype = 100; /* connect a pair */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ + graphmode.ptype = KAPA_POINT_PAIR_CONNECT; /* connect pairs of points */ graphmode.etype = 0; PlotVectorPair (kapa, &Xvec, &Yvec, NULL, &graphmode); Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/cline.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/cline.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/cline.c (revision 40477) @@ -20,5 +20,5 @@ /* set point style and errorbar mode (these are NOT sticky) */ - graphmode.style = 0; + graphmode.style = KAPA_PLOT_CONNECT; graphmode.etype = 0; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/czplot.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/czplot.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/czplot.c (revision 40477) @@ -111,5 +111,5 @@ Zvec.Nelements = Npts; - graphmode.style = 2; + graphmode.style = KAPA_PLOT_POINTS; /* points */ graphmode.size = -1; /* point size determined by Zvec */ graphmode.etype = 0; @@ -236,5 +236,5 @@ Zvec.Nelements = Npts; - graphmode.style = 2; + graphmode.style = KAPA_PLOT_POINTS; graphmode.color = -1; /* point color determined by Zvec */ graphmode.etype = 0; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitplx.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitplx.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitplx.c (revision 40477) @@ -77,5 +77,5 @@ double *dD = dDvec->elements.Flt; - int *mask = NULL; + opihi_int *mask = NULL; if (mvec) { mask = mvec->elements.Int; @@ -380,5 +380,5 @@ } -int PlxSetMeanEpoch (double *R, double *D, double *T, double *Rmean, double *Dmean, double *Tmean, int *mask, int Ntotal) { +int PlxSetMeanEpoch (double *R, double *D, double *T, double *Rmean, double *Dmean, double *Tmean, opihi_int *mask, int Ntotal) { int i; @@ -413,5 +413,5 @@ // generate the fit values (projected X,Y; parallax factors; -int PlxSetEpochPosition (PlxFitData *fitdata, double *R, double *D, double *dR, double *dD, double *T, int *mask, int Ntotal, Coords *coords, double Tmean) { +int PlxSetEpochPosition (PlxFitData *fitdata, double *R, double *D, double *dR, double *dD, double *T, opihi_int *mask, int Ntotal, Coords *coords, double Tmean) { int i; @@ -464,5 +464,5 @@ # define MAX_REJECT 0.1 -int PlxOutlierClip (PlxFitData *fitdata, int *mask, int Noutlier, float dPsigMax, Vector *dPvec, int VERBOSE) { +int PlxOutlierClip (PlxFitData *fitdata, opihi_int *mask, int Noutlier, float dPsigMax, Vector *dPvec, int VERBOSE) { int i, n; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitplx_irls.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitplx_irls.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitplx_irls.c (revision 40477) @@ -12,5 +12,4 @@ if ((mvec = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); remove_argument (N, &argc, argv); - CastVector (mvec, OPIHI_INT); } @@ -81,13 +80,21 @@ double *dD = dDvec->elements.Flt; - int *mask = NULL; - if (mvec) { - mask = mvec->elements.Int; - } - // Ntotal : all points supplied by user // Nsubset : unmasked points int Ntotal = tvec->Nelements; // XXX check other lengths - + + if (rvec->Nelements != Ntotal) ESCAPE ("mis-match in vector lengths (mjd vs ra) = (%d vs %d)\n", rvec->Nelements, Ntotal); + if (dvec->Nelements != Ntotal) ESCAPE ("mis-match in vector lengths (mjd vs dec) = (%d vs %d)\n", dvec->Nelements, Ntotal); + if (dRvec->Nelements != Ntotal) ESCAPE ("mis-match in vector lengths (mjd vs dR) = (%d vs %d)\n", dRvec->Nelements, Ntotal); + if (dDvec->Nelements != Ntotal) ESCAPE ("mis-match in vector lengths (mjd vs dD) = (%d vs %d)\n", dDvec->Nelements, Ntotal); + + opihi_int *mask = NULL; + if (mvec) { + ResetVector (mvec, OPIHI_INT, Ntotal); + mask = mvec->elements.Int; + for (i = 0; i < Ntotal; i++) { mask[i] = 1; } + } + + double Rmean, Dmean, Tmean; PlxSetMeanEpoch (R, D, T, &Rmean, &Dmean, &Tmean, mask, Ntotal); @@ -109,6 +116,6 @@ for (i = 0; (VERBOSE == 2) && (i < fitdata.Npts); i++) { int n = fitdata.index[i]; - int maskValue = mask ? mask[n] : 1; - fprintf (stderr, "%f %f : %f %d : %f %f %f\n", R[n], D[n], T[n], maskValue, fitdata.t[i], fitdata.X[i], fitdata.Y[i]); + opihi_int maskValue = mask ? mask[n] : 1; + fprintf (stderr, "%f %f : %f "OPIHI_INT_FMT" : %f %f %f\n", R[n], D[n], T[n], maskValue, fitdata.t[i], fitdata.X[i], fitdata.Y[i]); } @@ -135,29 +142,33 @@ // update the mask based on the input mask and the outlier limits. - double Sum_Wx = 0; - double Sum_Wy = 0; - mask = mvec->elements.Int; - for (i = 0; i < fitdata.Npts; i++) { - Sum_Wx += fitdata.Wx[i]; - Sum_Wy += fitdata.Wy[i]; - } - for (i = 0; i < fitdata.Npts; i++) { - // fitdata only includes the previously unmasked points - if ((fitdata.Wx[i] < outlier_limit * Sum_Wx / (1.0 * fitdata.Npts))|| - (fitdata.Wy[i] < outlier_limit * Sum_Wy / (1.0 * fitdata.Npts))) { - int n = fitdata.index[i]; - mask[n] = 0; + if (mask) { + double Sum_Wx = 0; + double Sum_Wy = 0; + + // calculate the total weight + for (i = 0; i < fitdata.Npts; i++) { + Sum_Wx += fitdata.Wx[i]; + Sum_Wy += fitdata.Wy[i]; + } + for (i = 0; i < fitdata.Npts; i++) { + // fitdata only includes the previously unmasked points + if ((fitdata.Wx[i] < outlier_limit * Sum_Wx / (1.0 * fitdata.Npts))|| + (fitdata.Wy[i] < outlier_limit * Sum_Wy / (1.0 * fitdata.Npts))) { + int n = fitdata.index[i]; + mask[n] = 0; - if (VERBOSE == 2) { - fprintf (stderr, "%f %f : %f %d : %f %f %f : %f %f %f %f\n", R[n], D[n], T[n], mask[n], fitdata.t[i], fitdata.X[i], fitdata.Y[i], fitdata.Wx[i], fitdata.Wy[i], Sum_Wx, Sum_Wy); - } - } - } - - if (Nresample){ - // now that the mask has been updated, we need to recalculate mean epoch and positions - // XXX make this conditional on actually masking unmasked points above - // PlxSetMeanEpoch (R, D, T, &Rmean, &Dmean, &Tmean, mask, Ntotal); - // PlxSetEpochPosition (&fitdata, R, D, dR, dD, T, mask, Ntotal, &coords, Tmean); + if (VERBOSE == 2) { + fprintf (stderr, "%f %f : %f "OPIHI_INT_FMT" : %f %f %f : %f %f %f %f\n", R[n], D[n], T[n], mask[n], fitdata.t[i], fitdata.X[i], fitdata.Y[i], fitdata.Wx[i], fitdata.Wy[i], Sum_Wx, Sum_Wy); + } + } + } + } + + if (Nresample) { + // if the mask has been updated, we need to recalculate mean epoch and positions + if (mask) { + PlxSetMeanEpoch (R, D, T, &Rmean, &Dmean, &Tmean, mask, Ntotal); + PlxSetEpochPosition (&fitdata, R, D, dR, dD, T, mask, Ntotal, &coords, Tmean); + } PlxFitData sample; @@ -218,7 +229,9 @@ // fprintf (stderr, "%f +/- %f | %f %f\n", fit.p, fit.dp, fit.uR, fit.uD); - Vector *dRresPMP, *dDresPMP, *dRresPLX, *dDresPLX; + Vector *dRresPOS, *dDresPOS, *dRresPMP, *dDresPMP, *dRresPLX, *dDresPLX; // save fit residuals (with only pm removed, and pm and plx removed) + if ((dRresPOS = SelectVector ("dRresPOS", ANYVECTOR, TRUE)) == NULL) ESCAPE ("cannot generate vector %s\n", "dRresPOS"); + if ((dDresPOS = SelectVector ("dDresPOS", ANYVECTOR, TRUE)) == NULL) ESCAPE ("cannot generate vector %s\n", "dDresPOS"); if ((dRresPMP = SelectVector ("dRresPMP", ANYVECTOR, TRUE)) == NULL) ESCAPE ("cannot generate vector %s\n", "dRresPMP"); if ((dDresPMP = SelectVector ("dDresPMP", ANYVECTOR, TRUE)) == NULL) ESCAPE ("cannot generate vector %s\n", "dDresPMP"); @@ -226,4 +239,6 @@ if ((dDresPLX = SelectVector ("dDresPLX", ANYVECTOR, TRUE)) == NULL) ESCAPE ("cannot generate vector %s\n", "dDresPLX"); + ResetVector (dRresPOS, OPIHI_FLT, Ntotal); + ResetVector (dDresPOS, OPIHI_FLT, Ntotal); ResetVector (dRresPMP, OPIHI_FLT, Ntotal); ResetVector (dDresPMP, OPIHI_FLT, Ntotal); @@ -246,4 +261,6 @@ double Yplx = fit.Do + fit.uD*t0; + dRresPOS->elements.Flt[i] = x0; + dDresPOS->elements.Flt[i] = y0; dRresPMP->elements.Flt[i] = x0 - Xpmp; dDresPMP->elements.Flt[i] = y0 - Ypmp; @@ -538,8 +555,9 @@ double wx,wy; + double lim_Wx = outlier_limit * Sum_Wx / (1.0 * Npts); + double lim_Wy = outlier_limit * Sum_Wy / (1.0 * Npts); for (i = 0; i < Npts; i++) { - if ((Wx[i] > outlier_limit * Sum_Wx / (1.0 * Npts))|| - (Wy[i] > outlier_limit * Sum_Wy / (1.0 * Npts))) { - + int skip = (Wx[i] < lim_Wx) || (Wy[i] < lim_Wy); + if (!skip) { Xf = fit[0].Ro + fit[0].uR*T[i] + fit[0].p*pR[i]; Yf = fit[0].Do + fit[0].uD*T[i] + fit[0].p*pD[i]; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitpm.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitpm.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitpm.c (revision 40477) @@ -50,5 +50,5 @@ double *dD = dDvec->elements.Flt; - int *mask = NULL; + opihi_int *mask = NULL; if (mvec) { mask = mvec->elements.Int; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitpm_irls.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitpm_irls.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/fitpm_irls.c (revision 40477) @@ -58,5 +58,5 @@ double *dD = dDvec->elements.Flt; - int *mask = NULL; + opihi_int *mask = NULL; if (mvec) { mask = mvec->elements.Int; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/init.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/init.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/init.c (revision 40477) @@ -33,4 +33,5 @@ int imfit PROTO((int, char **)); int imsub PROTO((int, char **)); +int jdtolst PROTO((int, char **)); int medianmap PROTO((int, char **)); int galsectors PROTO((int, char **)); @@ -103,4 +104,5 @@ {1, "imfit", imfit, "fit function"}, {1, "imsub", imsub, "subtract function"}, + {1, "jdtolst", jdtolst, "JD to LST conversion"}, {1, "medianmap", medianmap, "small median image"}, {1, "mkgauss", mkgauss, "generate a 2-D gaussian centered in image"}, Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/jdtolst.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/jdtolst.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/jdtolst.c (revision 40477) @@ -0,0 +1,90 @@ +# include "astro.h" + +int jdtolst (int argc, char **argv) { + + int N; + Vector *jdvec, *outvec; + + int VERBOSE = FALSE; + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + char *OutputName = NULL; + if ((N = get_argument (argc, argv, "-output"))) { + remove_argument (N, &argc, argv); + OutputName = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (argc != 2) goto syntax; + + char *long_str = get_variable ("LONGITUDE"); + if (!long_str) { + gprint (GP_ERR, "please define $LONGITUDE\n"); + FREE (OutputName); + return FALSE; + } + double longitude = atof (long_str); + if (VERBOSE) gprint (GP_ERR, "using longitude of %f\n", longitude); + free (long_str); + + if (ISNUM(argv[1][0])) { + + char *endptr; + double jd = strtod (argv[1], &endptr); + if (endptr == argv[1]) { + gprint (GP_ERR, "error interpretting %s as a number\n", argv[1]); + FREE (OutputName); + return FALSE; + } + + double lst = ohana_lst (jd, longitude); + + if (OutputName) { + set_variable (OutputName, lst); + FREE (OutputName); + return (TRUE); + } + + gprint (GP_ERR, "lst: %f\n", lst); + return (TRUE); + } + + if (!OutputName) { + gprint (GP_ERR, " if (JD) is a vector, -output must be supplied\n"); + return FALSE; + } + + if ((jdvec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) { + FREE (OutputName); + return FALSE; + } + + if (jdvec->type != OPIHI_FLT) { + gprint (GP_ERR, " (JD) must be of type FLOAT\n"); + FREE (OutputName); + return FALSE; + } + + if ((outvec = SelectVector (OutputName, ANYVECTOR, TRUE)) == NULL) { + FREE (OutputName); + return FALSE; + } + + ResetVector (outvec, OPIHI_FLT, jdvec[0].Nelements); + + for (int i = 0; i < jdvec[0].Nelements; i++) { + double lst = ohana_lst (jdvec->elements.Flt[i], longitude); + outvec->elements.Flt[i] = lst; + } + FREE (OutputName); + return TRUE; + + syntax: + gprint (GP_ERR, "USAGE: jdtolst (JD) [-output name]\n"); + gprint (GP_ERR, " (JD) may be a number or a vector.\n"); + gprint (GP_ERR, " if (JD) is a vector, (output) must be supplied and is used as the name of the output vector \n"); + return (FALSE); +} Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/star.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/star.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.astro/star.c (revision 40477) @@ -3,5 +3,5 @@ int star (int argc, char **argv) { - int x, y, N, dx, Nborder; + int x, y, N, Nborder; double max; Buffer *buf; @@ -33,6 +33,18 @@ } + int dx = 11; + int dy = 11; + int BOX = FALSE; + if ((N = get_argument (argc, argv, "-box"))) { + remove_argument (N, &argc, argv); + dx = atoi(argv[N]); + remove_argument (N, &argc, argv); + dy = atoi(argv[N]); + remove_argument (N, &argc, argv); + BOX = TRUE; + } + if ((argc != 4) && (argc != 5)) { - gprint (GP_ERR, "USAGE: star (buffer) x y [dx] [-border N] [-sat cnts]\n"); + gprint (GP_ERR, "USAGE: star (buffer) x y [dx] [-border N] [-sat cnts] [-box dx dy]\n"); gprint (GP_ERR, " dx is the aperture diameter, but is adjusted up to the next odd number\n"); return (FALSE); @@ -40,5 +52,4 @@ if ((buf = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); - dx = 11; x = atof (argv[2]); y = atof (argv[3]); @@ -47,5 +58,9 @@ } - get_aperture_stats (&buf[0].matrix, x, y, dx, Nborder, max, VERBOSE); + if (BOX) { + get_box_stats (&buf[0].matrix, x, y, dx, dy, Nborder, max, VERBOSE); + } else { + get_aperture_stats (&buf[0].matrix, x, y, dx, Nborder, max, VERBOSE); + } return (TRUE); Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/Makefile =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/Makefile (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/Makefile (revision 40477) @@ -66,4 +66,5 @@ $(SRC)/hermitian1d.$(ARCH).o \ $(SRC)/hermitian2d.$(ARCH).o \ +$(SRC)/idxread.$(ARCH).o \ $(SRC)/imcut.$(ARCH).o \ $(SRC)/imhist.$(ARCH).o \ @@ -99,7 +100,12 @@ $(SRC)/medimage_commands.$(ARCH).o \ $(SRC)/mset.$(ARCH).o \ -$(SRC)/needles.$(ARCH).o \ +$(SRC)/needles.$(ARCH).o \ +$(SRC)/nnet.$(ARCH).o \ +$(SRC)/nnet_commands.$(ARCH).o \ +$(SRC)/nnet_train.$(ARCH).o \ +$(SRC)/nnet_apply.$(ARCH).o \ $(SRC)/peak.$(ARCH).o \ $(SRC)/periodogram.$(ARCH).o \ +$(SRC)/periodogram-fm.$(ARCH).o \ $(SRC)/plot.$(ARCH).o \ $(SRC)/dot.$(ARCH).o \ @@ -154,4 +160,5 @@ $(SRC)/type.$(ARCH).o \ $(SRC)/uniq.$(ARCH).o \ +$(SRC)/uniqpair.$(ARCH).o \ $(SRC)/unsign.$(ARCH).o \ $(SRC)/vbin.$(ARCH).o \ @@ -159,4 +166,5 @@ $(SRC)/vclip.$(ARCH).o \ $(SRC)/vgauss.$(ARCH).o \ +$(SRC)/vlorentz.$(ARCH).o \ $(SRC)/vellipse.$(ARCH).o \ $(SRC)/vmaxwell.$(ARCH).o \ @@ -171,5 +179,5 @@ $(SRC)/vstats.$(ARCH).o \ $(SRC)/xsection.$(ARCH).o \ -$(SRC)/vsh.$(ARCH).o \ +$(SRC)/vsh.$(ARCH).o \ $(SRC)/vshfit.$(ARCH).o \ $(SRC)/shterms.$(ARCH).o \ @@ -187,4 +195,5 @@ $(INC)/external.h \ $(INC)/shell.h \ +$(INC)/data.h \ $(INC)/dvomath.h \ $(INC)/display.h Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/dot.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/dot.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/dot.c (revision 40477) @@ -29,5 +29,5 @@ /* set point style and errorbar mode (these are NOT sticky) */ - graphmode.style = 2; + graphmode.style = KAPA_PLOT_POINTS; graphmode.etype = 0; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/grid.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/grid.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/grid.c (revision 40477) @@ -175,6 +175,6 @@ Xvec.Nelements = Yvec.Nelements = N; - graphmode.style = 2; /* points */ - graphmode.ptype = 100; /* connect a pair */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ + graphmode.ptype = KAPA_POINT_PAIR_CONNECT; /* connect pairs of points */ graphmode.etype = 0; PlotVectorPair (kapa, &Xvec, &Yvec, NULL, &graphmode); Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/idxread.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/idxread.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/idxread.c (revision 40477) @@ -0,0 +1,176 @@ +# include "data.h" + +# undef SWAP_BYTE +# undef SWAP_WORD +# undef SWAP_DBLE + +# ifdef BYTE_SWAP +# define SWAP_BYTE(X) { \ + char tmp; char *tmp_Ptr = (char *) &X; \ + tmp = tmp_Ptr[0]; tmp_Ptr[0] = tmp_Ptr[1]; tmp_Ptr[1] = tmp; } +# define SWAP_WORD(X) { \ + char tmp; char *tmp_Ptr = (char *) &X; \ + tmp = tmp_Ptr[0]; tmp_Ptr[0] = tmp_Ptr[3]; tmp_Ptr[3] = tmp; \ + tmp = tmp_Ptr[1]; tmp_Ptr[1] = tmp_Ptr[2]; tmp_Ptr[2] = tmp; } +# define SWAP_DBLE(X) { \ + char tmp; char *tmp_Ptr = (char *) &X; \ + tmp = tmp_Ptr[0]; tmp_Ptr[0] = tmp_Ptr[7]; tmp_Ptr[7] = tmp; \ + tmp = tmp_Ptr[1]; tmp_Ptr[1] = tmp_Ptr[6]; tmp_Ptr[6] = tmp; \ + tmp = tmp_Ptr[2]; tmp_Ptr[2] = tmp_Ptr[5]; tmp_Ptr[5] = tmp; \ + tmp = tmp_Ptr[3]; tmp_Ptr[3] = tmp_Ptr[4]; tmp_Ptr[4] = tmp; } +# else +# define SWAP_BYTE(X) +# define SWAP_WORD(X) +# define SWAP_DBLE(X) +# endif + +typedef enum { IDX_NONE, IDX_UBYTE, IDX_SBYTE, IDX_SHORT, IDX_INT, IDX_FLOAT, IDX_DOUBLE} idxType; + +int idxread (int argc, char **argv) { + + int magic, Nread, byte_pix; + + if (argc != 3) { + gprint (GP_ERR, "USAGE: idxread (file) (basename)\n"); + gprint (GP_ERR, " the file may encode vectors, images, or higher dimension arrays.\n"); + gprint (GP_ERR, " if N(dim) <= 3, the basename will be used for the vector or (2D or 3D) image name.\n"); + gprint (GP_ERR, " if N(dim) >= 4, the basename will be used to generate 3D image names of the form basename_n_m.\n"); + return FALSE; + } + + FILE *f = fopen (argv[1], "r"); + if (!f) { + gprint (GP_ERR, "failed to open file %s\n", argv[1]); + return (FALSE); + } + + // read the magic int + Nread = fread (&magic, 4, 1, f); + if (!Nread) { + gprint (GP_ERR, "failed to read magic bytes from file %s\n", argv[1]); + fclose (f); + return (FALSE); + } + + SWAP_WORD (magic); + + char *myBytes = (char *) &magic; + if (myBytes[3] || myBytes[2]) { + gprint (GP_ERR, "invalid magic number 0x%08x\n", magic); + fclose (f); + return (FALSE); + } + + idxType type = IDX_NONE; + switch (myBytes[1]) { + case 0x08: type = IDX_UBYTE; byte_pix = 1; break; + case 0x09: type = IDX_SBYTE; byte_pix = 1; break; + case 0x0b: type = IDX_SHORT; byte_pix = 2; break; + case 0x0c: type = IDX_INT; byte_pix = 4; break; + case 0x0d: type = IDX_FLOAT; byte_pix = 4; break; + case 0x0e: type = IDX_DOUBLE; byte_pix = 8; break; + default: + gprint (GP_ERR, "invalid magic number 0x%08x (bad type)\n", magic); + fclose (f); + return (FALSE); + } + + int Ndim = myBytes[0]; + if ((Ndim < 1) || (Ndim > 9)) { + gprint (GP_ERR, "invalid magic number 0x%08x (bad Ndim)\n", magic); + fclose (f); + return (FALSE); + } + + int sizes[10]; + + // read the axis sizes + Nread = fread (&sizes, 4, Ndim, f); + if (Nread != Ndim) { + gprint (GP_ERR, "failed to read array sizes from file %s\n", argv[1]); + fclose (f); + return (FALSE); + } + + // read the full data array in one go? + + int Nbytes = byte_pix; + for (int i = 0; i < Ndim; i++) { + SWAP_WORD (sizes[i]); + Nbytes *= MAX (1, sizes[i]); + } + ALLOCATE_PTR (buffer, char, Nbytes); + + Nread = fread (buffer, 1, Nbytes, f); + fclose (f); + + if (Nread != Nbytes) { + gprint (GP_ERR, "failed to read array sizes from file %s\n", argv[1]); + return (FALSE); + } + + // generate a vector + if (Ndim == 1) { + Vector *vector; + if ((vector = SelectVector (argv[2], ANYVECTOR, TRUE)) == NULL) { + gprint (GP_ERR, "cannot use %s for a vector\n", argv[2]); + return FALSE; + } + + // this relies on the enum being ordered above + if (type < IDX_FLOAT) { + ResetVector (vector, OPIHI_INT, sizes[0]); + } else { + ResetVector (vector, OPIHI_FLT, sizes[0]); + } + + switch (type) { + case IDX_UBYTE: { unsigned char *ptr = (unsigned char *) buffer; for (int i = 0; i < sizes[0]; i++) { vector->elements.Int[i] = ptr[i]; } } break; + case IDX_SBYTE: { char *ptr = ( char *) buffer; for (int i = 0; i < sizes[0]; i++) { vector->elements.Int[i] = ptr[i]; } } break; + case IDX_SHORT: { short *ptr = ( short *) buffer; for (int i = 0; i < sizes[0]; i++) { SWAP_BYTE(ptr[i]); vector->elements.Int[i] = ptr[i]; } } break; + case IDX_INT: { int *ptr = ( int *) buffer; for (int i = 0; i < sizes[0]; i++) { SWAP_WORD(ptr[i]); vector->elements.Int[i] = ptr[i]; } } break; + case IDX_FLOAT: { float *ptr = ( float *) buffer; for (int i = 0; i < sizes[0]; i++) { SWAP_WORD(ptr[i]); vector->elements.Flt[i] = ptr[i]; } } break; + case IDX_DOUBLE: { double *ptr = ( double *) buffer; for (int i = 0; i < sizes[0]; i++) { SWAP_DBLE(ptr[i]); vector->elements.Flt[i] = ptr[i]; } } break; + default: + gprint (GP_ERR, "this should not happen\n"); + return FALSE; + } + } + + // generate a matrix + if ((Ndim == 2) || (Ndim == 3)) { + Buffer *matrix; + if ((matrix = SelectBuffer (argv[2], ANYBUFFER, TRUE)) == NULL) { + gprint (GP_ERR, "cannot use %s for a buffer\n", argv[2]); + return FALSE; + } + + gfits_free_matrix (&matrix[0].matrix); + gfits_free_header (&matrix[0].header); + + if (Ndim == 3) { + // dimension order is Nz, Ny, Nx + if (!CreateBuffer3D (matrix, sizes[2], sizes[1], sizes[0], -32, 1.0, 0.0)) return FALSE; + } else { + if (!CreateBuffer (matrix, sizes[1], sizes[0], -32, 1.0, 0.0)) return FALSE; + } + + float *out = (float *) matrix[0].matrix.buffer; + int Nvalue = sizes[0]*sizes[1]; + if (Ndim == 3) Nvalue *= sizes[2]; + + switch (type) { + case IDX_UBYTE: { unsigned char *ptr = (unsigned char *) buffer; for (int i = 0; i < Nvalue; i++) { out[i] = ptr[i]; } } break; + case IDX_SBYTE: { char *ptr = ( char *) buffer; for (int i = 0; i < Nvalue; i++) { out[i] = ptr[i]; } } break; + case IDX_SHORT: { short *ptr = ( short *) buffer; for (int i = 0; i < Nvalue; i++) { out[i] = ptr[i]; } } break; + case IDX_INT: { int *ptr = ( int *) buffer; for (int i = 0; i < Nvalue; i++) { out[i] = ptr[i]; } } break; + case IDX_FLOAT: { float *ptr = ( float *) buffer; for (int i = 0; i < Nvalue; i++) { out[i] = ptr[i]; } } break; + case IDX_DOUBLE: { double *ptr = ( double *) buffer; for (int i = 0; i < Nvalue; i++) { out[i] = ptr[i]; } } break; + default: + gprint (GP_ERR, "this should not happen\n"); + return FALSE; + } + } + + return TRUE; +} Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/impeaks.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/impeaks.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/impeaks.c (revision 40477) @@ -100,4 +100,5 @@ int Npeaks = 0; for (ix = 1; ix < Nx - 1; ix++) { + if (!isfinite(row[ix])) continue; // ignore NAN values if (row[ix] < threshold) continue; // only accept pixels above threshold if (row[ix] < row[ix - 1]) continue; // peak pixel must be at least preceeding pixel Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/init.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/init.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/init.c (revision 40477) @@ -55,4 +55,5 @@ int hermitian1d PROTO((int, char **)); int hermitian2d PROTO((int, char **)); +int idxread PROTO((int, char **)); int imcut PROTO((int, char **)); int imhist PROTO((int, char **)); @@ -89,6 +90,8 @@ int mset PROTO((int, char **)); int needles PROTO((int, char **)); +int nnet_command PROTO((int, char **)); int peak PROTO((int, char **)); int periodogram PROTO((int, char **)); +int periodogram_fm PROTO((int, char **)); int plot PROTO((int, char **)); int dot PROTO((int, char **)); @@ -140,6 +143,7 @@ int tvcontour PROTO((int, char **)); int tvgrid PROTO((int, char **)); -int opihi_type PROTO((int, char **)); +int opihi_type PROTO((int, char **)); int uniq PROTO((int, char **)); +int uniqpair PROTO((int, char **)); int unsign PROTO((int, char **)); int vbin PROTO((int, char **)); @@ -149,4 +153,5 @@ int vgrid PROTO((int, char **)); int vgauss PROTO((int, char **)); +int vlorentz PROTO((int, char **)); int vellipse PROTO((int, char **)); int vmaxwell PROTO((int, char **)); @@ -232,4 +237,5 @@ {1, "hermitian1d", hermitian1d, "generate 1-D Hermitian Polynomial"}, {1, "hermitian2d", hermitian2d, "generate 2-D Hermitian Polynomial"}, + {1, "idxread", idxread, "read vector or image data from an IDX file"}, {1, "imbin", rebin, "rebin image data by factor of N"}, {1, "imclip", imclip, "clip values in an image to be within a range"}, @@ -268,7 +274,9 @@ {1, "imset", mset, "insert a vector in an image"}, {1, "needles", needles, "plot vectors needles"}, + {1, "nnet", nnet_command, "Neural Network commands"}, {1, "parity", parity, "set image parity"}, {1, "peak", peak, "find vector peak in range"}, - {1, "periodogram", periodogram, "measure periods in unevenly sampled data"}, + {1, "periodogram", periodogram, "measure periods in unevenly sampled data (Lomb-Scargle)"}, + {1, "periodogram_fm", periodogram_fm, "measure periods in unevenly sampled data (generalized Lomb-Scargle; floating mean)"}, {1, "plot", plot, "plot a pair of vectors"}, {1, "png", jpeg, "convert display graphic to PNG"}, @@ -323,4 +331,5 @@ {1, "ungridify", ungridify, "convert image region to vector triplet"}, {1, "uniq", uniq, "create a uniq vector subset from a vector"}, + {1, "uniqpair", uniqpair, "create a uniq vector subset from a pair of vectors, saving duplicates if desired"}, {1, "unsign", unsign, "toggle the UNSIGN status"}, {1, "vbin", vbin, "rebin vector data by a factor of N"}, @@ -330,4 +339,5 @@ {1, "vtype", vtype, "return the vector type (FLT or INT)"}, {1, "vgauss", vgauss, "fit a Gaussian to a vector"}, + {1, "vlorentz", vlorentz, "fit a Lorentzian to a vector"}, {1, "vellipse", vellipse, "fit a Ellipse to a vector pair"}, {1, "vgrid", vgrid, "generate an image from a triplet of vectors"}, Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/limits.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/limits.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/limits.c (revision 40477) @@ -3,7 +3,6 @@ int limits (int argc, char **argv) { - int N, APPLY, dX, dY; + int N, dX, dY; int kapa; - char *name; Graphdata graphmode; Vector *xvec, *yvec; @@ -11,10 +10,56 @@ xvec = yvec = NULL; - APPLY = FALSE; + float minLimitX = NAN; + float minLimitY = NAN; + float maxLimitX = NAN; + float maxLimitY = NAN; + float delLimitX = NAN; + float delLimitY = NAN; + + if ((N = get_argument (argc, argv, "-minX"))) { + remove_argument (N, &argc, argv); + minLimitX = atof (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-maxX"))) { + remove_argument (N, &argc, argv); + maxLimitX = atof (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-delX"))) { + if (!isnan(minLimitX) || !isnan(maxLimitX)) { + gprint (GP_ERR, "-minX & -maxX cannot be mixed with -delX\n"); + return (FALSE); + } + remove_argument (N, &argc, argv); + delLimitX = atof (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-minY"))) { + remove_argument (N, &argc, argv); + minLimitY = atof (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-maxY"))) { + remove_argument (N, &argc, argv); + maxLimitY = atof (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-delY"))) { + if (!isnan(minLimitY) || !isnan(maxLimitY)) { + gprint (GP_ERR, "-minY & -maxY cannot be mixed with -delY\n"); + return (FALSE); + } + remove_argument (N, &argc, argv); + delLimitY = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + int APPLY = FALSE; if ((N = get_argument (argc, argv, "-a"))) { remove_argument (N, &argc, argv); APPLY = TRUE; } - name = NULL; + char *name = NULL; if ((N = get_argument (argc, argv, "-n"))) { remove_argument (N, &argc, argv); @@ -22,4 +67,5 @@ remove_argument (N, &argc, argv); } + if (!GetGraph (&graphmode, &kapa, name)) return (FALSE); FREE (name); @@ -97,4 +143,27 @@ success: SetLimits (xvec, yvec, &graphmode); + + if (!isnan(minLimitX)) graphmode.xmin = MIN (minLimitX, graphmode.xmin); + if (!isnan(maxLimitX)) graphmode.xmax = MAX (maxLimitX, graphmode.xmax); + if (!isnan(minLimitY)) graphmode.ymin = MIN (minLimitY, graphmode.ymin); + if (!isnan(maxLimitY)) graphmode.ymax = MAX (maxLimitY, graphmode.ymax); + + if (!isnan(delLimitX)) { + float delta = graphmode.xmax - graphmode.xmin; + if (fabs(delLimitX) > fabs(delta)) { + float midpt = 0.5*(graphmode.xmax + graphmode.xmin); + graphmode.xmax = midpt + 0.5*delLimitX; + graphmode.xmin = midpt - 0.5*delLimitX; + } + } + if (!isnan(delLimitY)) { + float delta = graphmode.ymax - graphmode.ymin; + if (fabs(delLimitY) > fabs(delta)) { + float midpt = 0.5*(graphmode.ymax + graphmode.ymin); + graphmode.ymax = midpt + 0.5*delLimitY; + graphmode.ymin = midpt - 0.5*delLimitY; + } + } + if (APPLY) KapaSetLimits (kapa, &graphmode); return (TRUE); Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/line.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/line.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/line.c (revision 40477) @@ -33,5 +33,5 @@ /* set point style and errorbar mode (these are NOT sticky) */ - graphmode.style = 0; + graphmode.style = KAPA_PLOT_CONNECT; graphmode.etype = 0; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/mslice.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/mslice.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/mslice.c (revision 40477) @@ -3,9 +3,33 @@ int mslice (int argc, char **argv) { - int i; + int N; Buffer *in, *out; + int Dir = 0; + if ((N = get_argument (argc, argv, "-x"))) { + remove_argument (N, &argc, argv); + Dir = 1; + } + if ((N = get_argument (argc, argv, "-y"))) { + remove_argument (N, &argc, argv); + if (Dir) { + gprint (GP_ERR, "ERROR: multiple -x,-y,-z options are not allowed\n"); + return FALSE; + } + Dir = 2; + } + if ((N = get_argument (argc, argv, "-z"))) { + remove_argument (N, &argc, argv); + if (Dir) { + gprint (GP_ERR, "ERROR: multiple -x,-y,-z options are not allowed\n"); + return FALSE; + } + Dir = 3; + } + if (!Dir) Dir = 3; + if (argc != 4) { - gprint (GP_ERR, "USAGE: mget plane\n"); + gprint (GP_ERR, "USAGE: mslice plane [-x,-y,-z]\n"); + gprint (GP_ERR, " -z is default\n"); return (FALSE); } @@ -24,22 +48,66 @@ int Nz = in[0].matrix.Naxis[2]; + int NplaneMax = 0; + int Nout1 = 0; + int Nout2 = 0; + switch (Dir) { + case 1: NplaneMax = Nx; Nout1 = Ny; Nout2 = Nz; break; + case 2: NplaneMax = Ny; Nout1 = Nx; Nout2 = Nz; break; + case 3: NplaneMax = Nz; Nout1 = Nx; Nout2 = Ny; break; + default: myAbort ("impossible"); + } + int invalid = FALSE; invalid = invalid || (plane < 0); - invalid = invalid || (plane >= Nz); + invalid = invalid || (plane >= NplaneMax); if (invalid) { - gprint (GP_ERR, "plane %d out of range\n", plane); + gprint (GP_ERR, "plane %d out of range (max = %d)\n", plane, NplaneMax); return (FALSE); } /* I should encapsulate this in a create_default_buffer */ + gfits_free_matrix (&out[0].matrix); gfits_free_header (&out[0].header); - if (!CreateBuffer (out, Nx, Ny, -32, 1.0, 0.0)) return FALSE; + if (!CreateBuffer (out, Nout1, Nout2, -32, 1.0, 0.0)) return FALSE; - float *inF = (float *) in[0].matrix.buffer + plane*Nx*Ny; - float *outF = (float *) out[0].matrix.buffer; + // pixel (ix, iy, iz) : in[0].matrix.buffer + ix + iy*Nx + iz*Nx*Ny - for (i = 0; i < Nx*Ny; i++, inF ++, outF++) { - *outF = *inF; + switch (Dir) { + case 1: { + float *outF = (float *) out[0].matrix.buffer; + + // pixel (ix, iy, iz) : in[0].matrix.buffer + ix + iy*Nx + iz*Nx*Ny + + for (int iz = 0; iz < Nz; iz++) { + float *inF = (float *) in[0].matrix.buffer + plane + iz*Nx*Ny; + for (int iy = 0; iy < Ny; iy++, inF += Nx, outF++) { + *outF = *inF; + } + } + break; + } + case 2: { + + float *outF = (float *) out[0].matrix.buffer; + + for (int iz = 0; iz < Nz; iz++) { + float *inF = (float *) in[0].matrix.buffer + plane*Nx + iz*Nx*Ny; + for (int ix = 0; ix < Nx; ix++, inF++, outF++) { + *outF = *inF; + } + } + break; + } + case 3: { + float *inF = (float *) in[0].matrix.buffer + plane*Nx*Ny; + float *outF = (float *) out[0].matrix.buffer; + + for (int i = 0; i < Nx*Ny; i++, inF ++, outF++) { + *outF = *inF; + } + break; + } + default: myAbort ("impossible"); } return (TRUE); Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet.c (revision 40477) @@ -0,0 +1,51 @@ +# include "data.h" + +static Command nnet_commands[] = { + {1, "list", nnet_list, "list nnets"}, + {1, "delete", nnet_delete, "delete a nnet"}, + {1, "print", nnet_print, "display nnet values"}, + {1, "create", nnet_create, "create a nnet"}, + {1, "set", nnet_set, "set nnet node values"}, + {1, "get", nnet_get, "get nnet node values"}, + {1, "read", nnet_read, "read nnet values from a file"}, + {1, "write", nnet_write, "write nnet values to a file"}, + {1, "train", nnet_train, "train nnet on a set of data"}, + {1, "apply", nnet_apply, "apply nnet to a set of data"}, +}; + +int nnet_command (int argc, char **argv) { + + int i, N, status; + + if (argc < 2) { + gprint (GP_ERR, "USAGE: nnet (command)\n"); + gprint (GP_ERR, " nnet list : list nnets\n"); + gprint (GP_ERR, " nnet delete (nnet) : delete a nnet\n"); + gprint (GP_ERR, " nnet print (nnet) : print values for a nnet\n"); + gprint (GP_ERR, " nnet create (nnet) (Ninput) [Nnodes] [Nnodes] ... (Noutput) : create a nnet\n"); + gprint (GP_ERR, " nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n"); + gprint (GP_ERR, " nnet get (nnet) [weights] [biases] ... [weights] [biases] : get nnet weights (images) and biases (vectors)\n"); + gprint (GP_ERR, " nnet read (nnet) (filename) : set nnet weights and biases using a data file\n"); + gprint (GP_ERR, " nnet write (nnet) (filename) : save nnet weights and biases to a data file\n"); + gprint (GP_ERR, " nnet train (nnet) [input] [input] ... [output] [output] ... : train nnet on data from a set of vectors\n"); + gprint (GP_ERR, " nnet apply (nnet) [input] [input] ... [output] [output] ... : apply nnet to input data and generate output\n"); + return (FALSE); + } + + N = sizeof (nnet_commands) / sizeof (Command); + + /* find the nnet sub-command which matches */ + for (i = 0; i < N; i++) { + if (!strcmp (nnet_commands[i].name, argv[1])) { + status = (*nnet_commands[i].func) (argc - 1, argv + 1); + return (status); + } + } + + gprint (GP_ERR, "unknown nnet command %s\n", argv[1]); + return (FALSE); +} + +/* nnet is called with the command "nnet". + the command line word "nnet" is meant to be followed the one of several + possible options list above */ Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_apply.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_apply.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_apply.c (revision 40477) @@ -0,0 +1,84 @@ +# include "data.h" + +int nnet_apply (int argc, char **argv) { + + if (argc < 4) { + gprint (GP_ERR, "USAGE: nnet apply (nnet) [input] [input] ... [output] [output] ...\n"); + return FALSE; + } + + Nnet *nnet = FindNnet (argv[1]); + if (nnet == NULL) { + gprint (GP_ERR, "nnet %s not found, create it first\n", argv[1]); + return FALSE; + } + + // save the number of input and output nodes; these need to match the supplied vectors + int Nlayer = nnet[0].Nlayer; + int Ninput = nnet[0].Nnodes[0]; + int Noutput = nnet[0].Nnodes[Nlayer - 1]; // number of input + output nodes + + // we need to have the right number of input and output vectors + if (argc != Ninput + Noutput + 2) { + gprint (GP_ERR, "need %d input and %d output vectors, but we have %d total\n", nnet[0].Nnodes[0], nnet[0].Nnodes[Nlayer - 1], argc - 2); + return FALSE; + } + + // grab the input and output vectors from the argument list + ALLOCATE_PTR (inVec, Vector *, Ninput); + ALLOCATE_PTR (outVec, Vector *, Noutput); + + // check that the vectors exist and that lengths match + int Ntrial = 0; + for (int i = 0; i < Ninput; i++) { + if ((inVec[i] = SelectVector (argv[i + 2], OLDVECTOR, FALSE)) == NULL) { + gprint (GP_ERR, "unknown input vector %s\n", argv[i+2]); + gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n"); + free (inVec); + free (outVec); + return (FALSE); + } + if (Ntrial && (inVec[i][0].Nelements != Ntrial)) { + gprint (GP_ERR, "input vectors have inconsistent lengths: %d vs %d for %s\n", Ntrial, inVec[i][0].Nelements, inVec[i][0].name); + free (inVec); + free (outVec); + return (FALSE); + } + if (!Ntrial) Ntrial = inVec[i][0].Nelements; + if (!Ntrial) { + gprint (GP_ERR, "trial vectors must be non-zero length\n"); + free (inVec); + free (outVec); + return (FALSE); + } + } + for (int i = 0; i < Noutput; i++) { + if ((outVec[i] = SelectVector (argv[i + 2 + Ninput], ANYVECTOR, FALSE)) == NULL) { + gprint (GP_ERR, "invalid output vector %s\n", argv[i+2+Ninput]); + gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n"); + free (inVec); + free (outVec); + return (FALSE); + } + ResetVector (outVec[i], OPIHI_FLT, Ntrial); + } + + for (int i = 0; i < Ntrial; i++) { + + // store the input values for this row (trial) in the input value vector + for (int j = 0; j < Ninput; j++) { + nnet[0].svalue[0][j] = inVec[j][0].elements.Flt[i]; + } + + nnet_feedforward (nnet); + + // store the output value vector values in output vectors for this row + for (int j = 0; j < Noutput; j++) { + outVec[j][0].elements.Flt[i] = nnet[0].svalue[Nlayer-1][j]; + } + } + + free (inVec); + free (outVec); + return TRUE; +} Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_commands.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_commands.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_commands.c (revision 40477) @@ -0,0 +1,375 @@ +# include "data.h" + +int nnet_list (int argc, char **argv) { + OHANA_UNUSED_PARAM(argv); + if (argc != 1) { + gprint (GP_ERR, "USAGE: nnet list\n"); + return FALSE; + } + + ListNnets(); + return TRUE; +} + +int nnet_delete (int argc, char **argv) { + + int status; + Nnet *nnet; + + if (argc != 2) { + gprint (GP_ERR, "USAGE: nnet delete (nnet)\n"); + return FALSE; + } + + nnet = FindNnet (argv[1]); + if (nnet == NULL) { + gprint (GP_ERR, "nnet %s not found\n", argv[1]); + return FALSE; + } + + status = DeleteNnet (nnet); + if (!status) abort (); + + return TRUE; +} + +int nnet_create (int argc, char **argv) { + + int N; + + int LargeWeightInit = FALSE; + if ((N = get_argument (argc, argv, "-large-weight-initializer"))) { + remove_argument (N, &argc, argv); + LargeWeightInit = TRUE; + } + + if (argc < 4) { + gprint (GP_ERR, "USAGE: nnet create (nnet) (Ninput) [Nnodes] [Nnodes] ... (Noutput)\n"); + return FALSE; + } + + int Nlayer = argc - 2; + + // nnet is guaranteed to be initialized with only name and the containers (but not vectors) allocated + Nnet *nnet = CreateNnet (argv[1], Nlayer); + + // define the size of each layer + for (int i = 0; i < Nlayer; i++) { + nnet[0].Nnodes[i] = atoi (argv[i + 2]); // input (0), hidden layer nodes, output (Nlayer) + } + + // allocate the vector and matrix data arrays (and init) + CreateNnetData (nnet, LargeWeightInit); + + return TRUE; +} + +int nnet_print (int argc, char **argv) { + + if (argc != 2) { + gprint (GP_ERR, "USAGE: nnet print (nnet)\n"); + return FALSE; + } + + Nnet *nnet = FindNnet (argv[1]); + if (nnet == NULL) { + gprint (GP_ERR, "nnet %s not found, create it first\n", argv[1]); + return FALSE; + } + + PrintNnet (nnet); + return TRUE; +} + +int nnet_set (int argc, char **argv) { + + Buffer *matrix; + Vector *vector; + + if (argc < 4) { + gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n"); + return FALSE; + } + + Nnet *nnet = FindNnet (argv[1]); + if (nnet == NULL) { + gprint (GP_ERR, "nnet %s not found, create it first\n", argv[1]); + return FALSE; + } + + // compare argc and nnet[0].Nlayer + if (argc - 2 != (nnet[0].Nlayer - 1) * 2) { + gprint (GP_ERR, "ERROR: invalid number of arguments\n"); + gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n"); + return FALSE; + } + + // check for existence of each matrix & vector (and check sizes) + for (int L = 1; L < nnet[0].Nlayer; L++) { + int Narg = 2*(L - 1) + 2; + if ((matrix = SelectBuffer (argv[Narg + 0], OLDBUFFER, FALSE)) == NULL) { + gprint (GP_ERR, "unknown input image %s\n", argv[Narg + 0]); + gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n"); + return (FALSE); + } + if ((vector = SelectVector (argv[Narg + 1], OLDVECTOR, FALSE)) == NULL) { + gprint (GP_ERR, "unknown input vector %s\n", argv[Narg + 1]); + gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n"); + return (FALSE); + } + + if (vector->Nelements != nnet[0].Nnodes[L]) { + gprint (GP_ERR, "invalid bias vector length %s (%d vs %d)\n", argv[Narg + 1], vector->Nelements, nnet[0].Nnodes[L]); + gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n"); + return (FALSE); + } + + if (matrix->matrix.Naxis[0] != nnet[0].Nnodes[L-1]) { + gprint (GP_ERR, "invalid weight image size for %s (Nx = %d vs %d)\n", argv[Narg + 0], (int) matrix->matrix.Naxis[0], nnet[0].Nnodes[L-1]); + gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n"); + return (FALSE); + } + + if (matrix->matrix.Naxis[1] != nnet[0].Nnodes[L]) { + gprint (GP_ERR, "invalid weight image size for %s (Ny = %d vs %d)\n", argv[Narg + 0], (int) matrix->matrix.Naxis[1], nnet[0].Nnodes[L]); + gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n"); + return (FALSE); + } + + } + + for (int L = 1; L < nnet[0].Nlayer; L++) { + + int Narg = 2*(L - 1) + 2; + matrix = SelectBuffer (argv[Narg + 0], OLDBUFFER, FALSE); // I've checked the argv entry above so I do not need to check again... + vector = SelectVector (argv[Narg + 1], OLDVECTOR, FALSE); + + opihi_flt *vecvalue = vector[0].elements.Flt; + for (int i = 0; i < nnet[0].Nnodes[L]; i++) { + nnet[0].biases[L][i] = vecvalue[i]; + } + + float *matvalue = (float *) matrix[0].matrix.buffer; + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) { + int k = i + j*nnet[0].Nnodes[L-1]; + nnet[0].weight[L][k] = matvalue[k]; + } + } + + } + + return TRUE; +} + +int nnet_read (int argc, char **argv) { + + # define D_LINE 0x10000 + char word[128]; + char line[D_LINE]; + + if (argc < 3) { + gprint (GP_ERR, "USAGE: nnet read (nnet) (filename) : set nnet weights and biases based on a file\n"); + gprint (GP_ERR, "the first line of the file specifies the number of layers, each layer in the network is written as a matrix of numbers (weights) and a vector\n"); + return FALSE; + } + + // open the file and read the number of layers + FILE *f = fopen (argv[2], "r"); + if (f == NULL) { + gprint (GP_ERR, "USAGE: nnet read (nnet) (filename) : set nnet weights and biases based on a file\n"); + gprint (GP_ERR, "file %s could not be opened\n", argv[2]); + return FALSE; + } + + // read the number of layers + // NLAYER 3 + int Nlayer; + scan_line_maxlen (f, line, D_LINE); + sscanf (line, "%127s %d", word, &Nlayer); + if (strcmp(word, "NLAYER")) { + gprint (GP_ERR, "warning: NLAYER keyword not found\n"); + } + + Nnet *nnet = CreateNnet (argv[1], Nlayer); + + // read the number of nodes + // LAYERS 2 4 2 + scan_line_maxlen (f, line, D_LINE); + char *tmpword = getword (line); + if (strcmp (tmpword, "LAYERS")) { + gprint (GP_ERR, "warning: LAYERS keyword not found\n"); + } + FREE (tmpword); + for (int i = 0; i < Nlayer; i++) { + int Nnode; + int status = iparse (&Nnode, i + 2, line); // numbering is fields 1 2 3 + if (!status) { + gprint (GP_ERR, "error: failed to find all Nnode values\n"); + fclose (f); + DeleteNnet (nnet); + return FALSE; + } + nnet[0].Nnodes[i] = Nnode; + } + + // this creates the data for each node and inits with gaussian weights + CreateNnetData (nnet, FALSE); + + // read the weights and biases from the data file + for (int L = 1; L < nnet[0].Nlayer; L++) { + + char word1[128], word2[128], word3[128]; + int Nx, Ny, layer; + scan_line_maxlen (f, line, D_LINE); + sscanf (line, "%127s %d %127s %d %127s %d", word1, &layer, word2, &Nx, word3, &Ny); + gprint (GP_ERR, "LAYER: %d, Nx: %d, Ny: %d\n", layer, Nx, Ny); + + if (layer != L - 1) { + gprint (GP_ERR, "warning: expect layer = %d, got %d\n", L - 1, layer); + } + if (Nx != nnet[0].Nnodes[L - 1]) { + gprint (GP_ERR, "warning: expect Nx = %d, got %d\n", nnet[0].Nnodes[L - 1], Nx); + } + if (Ny != nnet[0].Nnodes[L]) { + gprint (GP_ERR, "warning: expect Ny = %d, got %d\n", nnet[0].Nnodes[L], Ny); + } + + double value; + for (int j = 0; j < Ny; j++) { + scan_line_maxlen (f, line, D_LINE); + for (int i = 0; i < Nx; i++) { + int k = i + j*nnet[0].Nnodes[L-1]; + int status = dparse (&value, i + 1, line); + if (!status) { + gprint (GP_ERR, "error: failed to read entry from a line (layer: %d, i: %d, j: %d)\n", layer, i, j); + fclose (f); + DeleteNnet (nnet); + return FALSE; + } + nnet[0].weight[L][k] = value; + } + dparse (&value, Nx + 1, line); + nnet[0].biases[L][j] = value; + } + } + + return TRUE; +} + +int nnet_write (int argc, char **argv) { + + if (argc < 3) { + gprint (GP_ERR, "USAGE: nnet write (nnet) (filename) : save nnet weights and biases to a file\n"); + return FALSE; + } + + Nnet *nnet = FindNnet (argv[1]); + if (nnet == NULL) { + gprint (GP_ERR, "nnet %s not found, create it first\n", argv[1]); + return FALSE; + } + + // open the file and read the number of layers + FILE *f = fopen (argv[2], "w"); + if (f == NULL) { + gprint (GP_ERR, "USAGE: nnet write (nnet) (filename) : save nnet weights and biases to a file\n"); + gprint (GP_ERR, "file %s could not be opened\n", argv[2]); + return FALSE; + } + + // write the number of layers + // NLAYER 3 + fprintf (f, "NLAYER %d\n", nnet->Nlayer); + + // write the number of nodes for each layer + // LAYERS 2 4 2 + fprintf (f, "LAYERS"); + for (int i = 0; i < nnet->Nlayer; i++) { + fprintf (f, " %d", nnet->Nnodes[i]); + } + fprintf (f, "\n"); + + // read the weights and biases from the data file + for (int L = 1; L < nnet[0].Nlayer; L++) { + + fprintf (f, "LAYER %d NX %d NY %d\n", L - 1, nnet->Nnodes[L-1], nnet->Nnodes[L]); + + for (int j = 0; j < nnet->Nnodes[L]; j++) { + for (int i = 0; i < nnet->Nnodes[L-1]; i++) { + int k = i + j*nnet[0].Nnodes[L-1]; + fprintf (f, "%f ", nnet[0].weight[L][k]); + } + fprintf (f, "%f\n", nnet[0].biases[L][j]); + } + } + fclose (f); + + return TRUE; +} + +int nnet_get (int argc, char **argv) { + + Buffer *matrix; + Vector *vector; + + if (argc < 4) { + gprint (GP_ERR, "USAGE: nnet get (nnet) [weights] [biases] ... [weights] [biases] : get nnet weights (images) and biases (vectors)\n"); + return FALSE; + } + + Nnet *nnet = FindNnet (argv[1]); + if (nnet == NULL) { + gprint (GP_ERR, "nnet %s not found, create it first\n", argv[1]); + return FALSE; + } + + // compare argc and nnet[0].Nlayer + if (argc - 2 != (nnet[0].Nlayer - 1) * 2) { + gprint (GP_ERR, "ERROR: invalid number of arguments\n"); + gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n"); + return FALSE; + } + + // check for existence of each matrix & vector (and check sizes) + for (int L = 1; L < nnet[0].Nlayer; L++) { + int Narg = 2*(L - 1) + 2; + if ((matrix = SelectBuffer (argv[Narg + 0], ANYBUFFER, FALSE)) == NULL) { + gprint (GP_ERR, "unknown input image %s\n", argv[Narg + 0]); + gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n"); + return (FALSE); + } + if ((vector = SelectVector (argv[Narg + 1], ANYVECTOR, FALSE)) == NULL) { + gprint (GP_ERR, "unknown input vector %s\n", argv[Narg + 1]); + gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n"); + return (FALSE); + } + } + + for (int L = 1; L < nnet[0].Nlayer; L++) { + + int Narg = 2*(L - 1) + 2; + matrix = SelectBuffer (argv[Narg + 0], OLDBUFFER, FALSE); // I've checked the argv entry above so I do not need to check again... + vector = SelectVector (argv[Narg + 1], OLDVECTOR, FALSE); + + ResetVector (vector, OPIHI_FLT, nnet[0].Nnodes[L]); + ResetBuffer (matrix, nnet[0].Nnodes[L-1], nnet[0].Nnodes[L], -32, 0.0, 1.0); + + opihi_flt *vecvalue = vector[0].elements.Flt; + for (int i = 0; i < nnet[0].Nnodes[L]; i++) { + vecvalue[i] = nnet[0].biases[L][i]; + } + + float *matvalue = (float *) matrix[0].matrix.buffer; + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) { + int k = i + j*nnet[0].Nnodes[L-1]; + matvalue[k] = nnet[0].weight[L][k]; + } + } + + } + + return TRUE; +} + Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_train.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_train.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_train.c (revision 40477) @@ -0,0 +1,531 @@ +# include "data.h" + +// these are local only +float nnet_sigmoid (float value); +void nnet_sortseq (float *X, int *Y, int N); +int nnet_onelayer (Nnet *nnet, int L); +void nnet_reset_Nabla (Nnet *nnet); +void nnet_update_Nabla (Nnet *nnet); +void nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta, float lambda, int Ntrial); +void nnet_backprop (Nnet *nnet, Vector **inVec, Vector **outVec, int N); +void nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta, float lambda); +void nnet_print_Nabla (Nnet *nnet); +void nnet_write_Nabla (char *filename, Nnet *nnet); + +static int QUADRATIC_COST = 0; + +int nnet_train (int argc, char **argv) { + + int N; + + int Nepoch = 10; + if ((N = get_argument (argc, argv, "-Nepoch"))) { + remove_argument (N, &argc, argv); + Nepoch = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + int Nmini = 100; + if ((N = get_argument (argc, argv, "-Nmini"))) { + remove_argument (N, &argc, argv); + Nmini = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + float eta = 1.0; // XXX how do I set this? does it need to update? + if ((N = get_argument (argc, argv, "-eta"))) { + remove_argument (N, &argc, argv); + eta = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + float lambda = 0.0; // XXX how do I set this? does it need to update? + if ((N = get_argument (argc, argv, "-lambda"))) { + remove_argument (N, &argc, argv); + lambda = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + Vector *resid = NULL; + if ((N = get_argument (argc, argv, "-resid"))) { + remove_argument (N, &argc, argv); + if ((resid = SelectVector (argv[N], ANYVECTOR, FALSE)) == NULL) { + gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n"); + gprint (GP_ERR, "cannot assign residual vector %s\n", argv[N]); + return FALSE; + } + remove_argument (N, &argc, argv); + } + + Buffer *result = NULL; + if ((N = get_argument (argc, argv, "-result"))) { + remove_argument (N, &argc, argv); + if ((result = SelectBuffer (argv[N], ANYBUFFER, FALSE)) == NULL) { + gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n"); + gprint (GP_ERR, "cannot assign result image %s\n", argv[N]); + return FALSE; + } + remove_argument (N, &argc, argv); + } + + QUADRATIC_COST = 0; + if ((N = get_argument (argc, argv, "-quadratic-cost"))) { + remove_argument (N, &argc, argv); + QUADRATIC_COST = 1; + } + + if (argc < 4) { + gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n"); + gprint (GP_ERR, "OPTIONS: -Nepoch [N] -Nmini [N]\n"); + // FREE (resid); + return FALSE; + } + + Nnet *nnet = FindNnet (argv[1]); + if (nnet == NULL) { + gprint (GP_ERR, "nnet %s not found, create it first\n", argv[1]); + // FREE (resid); + return FALSE; + } + + // save the number of input and output nodes; these need to match the supplied vectors + int Nlayer = nnet[0].Nlayer; + int Ninput = nnet[0].Nnodes[0]; + int Noutput = nnet[0].Nnodes[Nlayer - 1]; // number of input + output nodes + + // we need to have the right number of input and output vectors + if (argc != Ninput + Noutput + 2) { + gprint (GP_ERR, "need %d input and %d output vectors, but we have %d total\n", nnet[0].Nnodes[0], nnet[0].Nnodes[Nlayer - 1], argc - 2); + // FREE (resid); + return FALSE; + } + + // grab the input and output vectors from the argument list + ALLOCATE_PTR (inVec, Vector *, Ninput); + ALLOCATE_PTR (outVec, Vector *, Noutput); + + // check that the vectors exist and that lengths match + int Ntrial = 0; + for (int i = 0; i < Ninput; i++) { + if ((inVec[i] = SelectVector (argv[i + 2], OLDVECTOR, FALSE)) == NULL) { + gprint (GP_ERR, "unknown input vector %s\n", argv[i+2]); + gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n"); + free (inVec); + free (outVec); + // FREE (resid); + return FALSE; + } + if (Ntrial && (inVec[i][0].Nelements != Ntrial)) { + gprint (GP_ERR, "input vectors have inconsistent lengths: %d vs %d for %s\n", Ntrial, inVec[i][0].Nelements, inVec[i][0].name); + free (inVec); + free (outVec); + // FREE (resid); + return FALSE; + } + if (!Ntrial) Ntrial = inVec[i][0].Nelements; + if (!Ntrial) { + gprint (GP_ERR, "trial vectors must be non-zero length\n"); + free (inVec); + free (outVec); + // FREE (resid); + return FALSE; + } + } + for (int i = 0; i < Noutput; i++) { + if ((outVec[i] = SelectVector (argv[i + 2 + Ninput], OLDVECTOR, FALSE)) == NULL) { + gprint (GP_ERR, "unknown output vector %s\n", argv[i+2+Ninput]); + gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n"); + free (inVec); + free (outVec); + // FREE (resid); + return FALSE; + } + if (outVec[i][0].Nelements != Ntrial) { + gprint (GP_ERR, "output and input vectors have inconsistent lengths: %d vs %d for %s\n", Ntrial, outVec[i][0].Nelements, outVec[i][0].name); + free (inVec); + free (outVec); + // FREE (resid); + return FALSE; + } + } + + // core of the training process: + + // index to hold a random sequence + ALLOCATE_PTR (seq, int, Ntrial); + ALLOCATE_PTR (rnd, float, Ntrial); + + if (resid) ResetVector (resid, OPIHI_FLT, Nepoch); + + if (1) { + int Npts = 0; + float s1 = 0.0, s2 = 0.0; + for (int i = 0; i < Ntrial; i++) { + + // store the input values for this row (trial) in the input value vector + for (int j = 0; j < Ninput; j++) { + nnet[0].svalue[0][j] = inVec[j][0].elements.Flt[i]; + } + + nnet_feedforward (nnet); + + // store the output value vector values in output vectors for this row + for (int j = 0; j < Noutput; j++) { + float dS = outVec[j][0].elements.Flt[i] - nnet[0].svalue[Nlayer-1][j]; + s1 += dS; + s2 += SQ(dS); + Npts ++; + } + } + float mean = s1 / Npts; + float sigma = sqrt(s2 / Npts - mean*mean); + gprint (GP_ERR, "start, %f +/- %f\n", mean, sigma); + } + + // PrintNnet (nnet); + + // train for Nepochs + // this recreates 'SGD' from http://neuralnetworksanddeeplearning.com/chap1.html + for (int epoch = 0; epoch < Nepoch; epoch ++) { + + // generate a random sequence : used to select random mini batches + for (int i = 0; i < Ntrial; i++) { seq[i] = i; rnd[i] = drand48(); } + nnet_sortseq (rnd, seq, Ntrial); + + int Npass = Ntrial / Nmini; + for (int pass = 0; pass < Npass; pass ++) { + // for a given pass select seq elements pass*Nmini to pass*Nmini + Nmini - 1 + // update the weights and biases using the mini batch subset + nnet_descent_step (nnet, inVec, outVec, seq, pass, Nmini, eta, lambda); + // return TRUE; // XXX short-circuit at one step + } + // PrintNnet (nnet); + + if (resid) { + int Npts = 0; + float s1 = 0.0, s2 = 0.0; + for (int i = 0; i < Ntrial; i++) { + + // store the input values for this row (trial) in the input value vector + for (int j = 0; j < Ninput; j++) { + nnet[0].svalue[0][j] = inVec[j][0].elements.Flt[i]; + } + + nnet_feedforward (nnet); + + // store the output value vector values in output vectors for this row + for (int j = 0; j < Noutput; j++) { + float dS = outVec[j][0].elements.Flt[i] - nnet[0].svalue[Nlayer-1][j]; + s1 += dS; + s2 += SQ(dS); + Npts ++; + } + } + float mean = s1 / Npts; + float sigma = sqrt(s2 / Npts - mean*mean); + // if (epoch % 10 == 0) gprint (GP_ERR, "epoch %d of %d, %f +/- %f\n", epoch, Nepoch, mean, sigma); + gprint (GP_ERR, "epoch %d of %d, %f +/- %f\n", epoch, Nepoch, mean, sigma); + resid[0].elements.Flt[epoch] = sigma; + } else { + // if (epoch % 10 == 0) gprint (GP_ERR, "epoch %d of %d\n", epoch, Nepoch); + gprint (GP_ERR, "epoch %d of %d\n", epoch, Nepoch); + } + } + // PrintNnet (nnet); + + if (result) { + + gfits_free_matrix (&result[0].matrix); + gfits_free_header (&result[0].header); + if (!CreateBuffer (result, Noutput, Ntrial, -32, 1.0, 0.0)) return FALSE; + + + float *value = (float *) result[0].matrix.buffer; + + for (int i = 0; i < Ntrial; i++) { + // store the input values for this row (trial) in the input value vector + for (int j = 0; j < Ninput; j++) { + nnet[0].svalue[0][j] = inVec[j][0].elements.Flt[i]; + } + + nnet_feedforward (nnet); + + for (int j = 0; j < Noutput; j++) { + value[j + i*Noutput] = nnet[0].svalue[Nlayer-1][j]; + } + } + } + + free (seq); + free (rnd); + free (inVec); + free (outVec); + return TRUE; +} + +// this recreates 'update_mini_batch' from http://neuralnetworksanddeeplearning.com/chap1.html +void nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta, float lambda) { + + int Ntrial = inVec[0][0].Nelements; + + nnet_reset_Nabla (nnet); + + for (int i = 0; (i < Nmini) && (pass*Nmini + i < Ntrial); i++) { + + // N is the element of the mini batch on which we are currently operating + // int N = seq[pass*Nmini + i]; // XXX uncomment to turn on random shuffle + int N = pass*Nmini + i; + + // backprop generates a dNabla_b, dNabla_w pair for the element N of the input and output vectors + nnet_backprop (nnet, inVec, outVec, N); + nnet_update_Nabla (nnet); + // gprint (GP_ERR, ". "); + + // nnet_print_Nabla (nnet); // XXX print nabla for each epoch + // XXX uncomment to dump nablas after one step, one element + // nnet_write_Nabla ("test.nabla.op.dat", nnet); // XXX print nabla for each epoch + // return; + } + // gprint (GP_ERR, " done mini batch\n"); + + // nnet_print_Nabla (nnet); + nnet_apply_Nabla (nnet, Nmini, eta, lambda, Ntrial); + + // XXX uncomment to dump nablas after one mini batch + // nnet_write_Nabla ("test.nabla.op.dat", nnet); // XXX print nabla for each epoch + // PrintNnet (nnet); +} + +void nnet_backprop (Nnet *nnet, Vector **inVec, Vector **outVec, int N) { + + // start with the input values + int Nlayer = nnet[0].Nlayer; + + // store the input values for this row (trial) in the input vector "svalue[0]" + for (int j = 0; j < nnet[0].Nnodes[0]; j++) { + nnet[0].svalue[0][j] = inVec[j][0].elements.Flt[N]; + } + + // z = w * activation + bias + // save z [one per non-input layer] + // activation = sigmoid (z) + + // feedforward operates on the vector saved in svalue[0] + // feedforward saves the zvalues [w * input + bias], svalues [sigmoid(z)], sprimes [sigmoid'(z)] as it runs + nnet_feedforward (nnet); + + // backward pass + for (int L = Nlayer - 1; L > 0; L--) { + + if (L == Nlayer - 1) { + // starting point uses cost_derivative to compare last svalue set with truth output + // delta = cost_derivative(svalue[L], output) * sprime[L]; + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + // cost_derivative (svalue[L][j], output[j]) = svalue[L][j] - output[j] + // (because cost = (1/2)(svalue[L][j] - output[j])^2) + // NOTE: the (1/Npt) factor is pushed to the apply_Nabla step + // nnet[0].delta[L][j] = cost_derivative(nnet[0].svalue[L][j], outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j]; + + if (QUADRATIC_COST) { + nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j]; + } else { + nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]); + } + + // for quadratic cost, delta = (svalue[L] - output) * sprime[L] + // for cross-entropy, delta = (svalue[L] - output) + + } + } else { + // XXX TEST PRINTS to catch code errors compared to python implementation + // gprint (GP_ERR, "z: "); + // for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + // gprint (GP_ERR, "%f ", nnet[0].zvalue[L][j]); + // } gprint (GP_ERR, "\n"); + // gprint (GP_ERR, "sp: "); + // for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + // gprint (GP_ERR, "%f ", nnet[0].sprime[L][j]); + // } gprint (GP_ERR, "\n"); + + // delta = DOT(delta, transpose(weight[L+1])) * sprime; + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + float tmpdelta = 0.0; + for (int i = 0; i < nnet[0].Nnodes[L+1]; i++) { + int k = j + i*nnet[0].Nnodes[L]; // note order of (i,j) : j is [L+1] direction + myAssert (k < nnet[0].Nnodes[L]*nnet[0].Nnodes[L+1], "overflow"); + tmpdelta += nnet[0].weight[L+1][k] * nnet[0].delta[L+1][i]; + // gprint (GP_ERR, "%e %e\n", nnet[0].weight[L+1][k], nnet[0].delta[L+1][i]); + } + nnet[0].delta[L][j] = tmpdelta * nnet[0].sprime[L][j]; + } + } + + // NOTE on weight array: for a given layer, L, the matrix weight[L] maps the nodes in + // the previous layer (L-1) to those in L: Nnodes[L-1] -> Nnodes[L]. This is a matrix + // with dimensions (Nnodes[L-1] x Nnodes[L]). The Nnodes[L-1] is the fast dimension, + // so for an element (i,j), the index k = i + j*Nnodes[L-1]. + + // Nabla_b[L] = delta; + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + nnet[0]. dNabla_b[L][j] = nnet[0].delta[L][j]; + } + + // Nabla_w[L] = DOT(delta, transpose(svalue[L-1])); + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) { + int k = i + j*nnet[0].Nnodes[L-1]; + myAssert (k < nnet[0].Nnodes[L-1]*nnet[0].Nnodes[L], "overflow"); + nnet[0]. dNabla_w[L][k] = nnet[0].svalue[L-1][i] * nnet[0].delta[L][j]; + } + } + } +} + +// support functions to loop over the Nabla entries +void nnet_reset_Nabla (Nnet *nnet) { + for (int L = 1; L < nnet[0].Nlayer; L++) { + + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + + nnet[0]. Nabla_b[L][j] = 0; + nnet[0].dNabla_b[L][j] = 0; + + for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) { + int k = i + j*nnet[0].Nnodes[L-1]; + myAssert (k < nnet[0].Nnodes[L-1]*nnet[0].Nnodes[L], "overflow"); + nnet[0]. Nabla_w[L][k] = 0; + nnet[0].dNabla_w[L][k] = 0; + } + } + } +} +void nnet_update_Nabla (Nnet *nnet) { + for (int L = 1; L < nnet[0].Nlayer; L++) { + + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + nnet[0]. Nabla_b[L][j] += nnet[0].dNabla_b[L][j]; + + for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) { + int k = i + j*nnet[0].Nnodes[L-1]; + myAssert (k < nnet[0].Nnodes[L-1]*nnet[0].Nnodes[L], "overflow"); + nnet[0]. Nabla_w[L][k] += nnet[0].dNabla_w[L][k]; + } + } + } +} +void nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta, float lambda, int Ntrial) { + for (int L = 1; L < nnet[0].Nlayer; L++) { + + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + nnet[0].biases[L][j] -= (eta / Nmini) * nnet[0].Nabla_b[L][j]; + + for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) { + int k = i + j*nnet[0].Nnodes[L-1]; + myAssert (k < nnet[0].Nnodes[L-1]*nnet[0].Nnodes[L], "overflow"); + // nnet[0].weight[L][k] -= (eta / Nmini) * nnet[0].Nabla_w[L][k]; + // with lambda > 0.0, we have L2 regularization. if lambda = 0.0, we recover the default implementation + nnet[0].weight[L][k] = nnet[0].weight[L][k]*(1.0 - eta*lambda/Ntrial) - (eta / Nmini) * nnet[0].Nabla_w[L][k]; + } + } + } +} + +void nnet_print_Nabla (Nnet *nnet) { + + for (int L = 1; L < nnet[0].Nlayer; L++) { + gprint (GP_ERR, " ----- Nabla %d -----\n", L); + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) { + int k = j * nnet[0].Nnodes[L-1] + i; + myAssert (k < nnet[0].Nnodes[L-1]*nnet[0].Nnodes[L], "overflow"); + gprint (GP_ERR, "%10.3e ", nnet[0].Nabla_w[L][k]); + } + gprint (GP_ERR, " : %10.3e\n", nnet[0].Nabla_b[L][j]); + } + } + return; +} + +void nnet_write_Nabla (char *filename, Nnet *nnet) { + + FILE *f = fopen (filename, "w"); + + fprintf (f, "NLAYER %d\n", nnet[0].Nlayer); + fprintf (f, "LAYERS "); + for (int L = 0; L < nnet[0].Nlayer; L++) { + fprintf (f, "%d ", nnet[0].Nnodes[L]); + } + fprintf (f, "\n"); + + for (int L = 1; L < nnet[0].Nlayer; L++) { + fprintf (f, "LAYER %d NX %d NY %d\n", L - 1, nnet[0].Nnodes[L-1], nnet[0].Nnodes[L]); + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) { + int k = j * nnet[0].Nnodes[L-1] + i; + myAssert (k < nnet[0].Nnodes[L-1]*nnet[0].Nnodes[L], "overflow"); + fprintf (f, "%.9f ", nnet[0].Nabla_w[L][k]); + } + fprintf (f, "%.9f\n", nnet[0].Nabla_b[L][j]); + } + } + + fclose (f); + return; +} + +// the input values must already be copied to the input layer svalue[] +void nnet_feedforward (Nnet *nnet) { + + for (int i = 1; i < nnet[0].Nlayer; i++) { + nnet_onelayer (nnet, i); + } + return; +} + +// calculate z, sigmoid(z) for each layer (z = w*value + bias) +int nnet_onelayer (Nnet *nnet, int L) { + + if (L < 1) return FALSE; // abort here? + if (L >= nnet[0].Nlayer) return FALSE; // abort here? + + // evaluating a single layer [L], n > 0, n < Nlayer: + int Ninput = nnet[0].Nnodes[L-1]; + int Noutput = nnet[0].Nnodes[L]; + + // input layer is [L-1], output layer is [L] + for (int j = 0; j < Noutput; j ++) { + float sum = 0; + for (int i = 0; i < Ninput; i++) { + // weight matrix order is (0, 1, ... Ninput-1, Ninput, Ninput + 1, ... Ninput * Noutput - 1) + int k = j * Ninput + i; + myAssert (k < Ninput*Noutput, "overflow"); + sum += nnet[0].weight[L][k]*nnet[0].svalue[L-1][i]; + } + sum += nnet[0].biases[L][j]; + nnet[0].zvalue[L][j] = sum; + nnet[0].svalue[L][j] = nnet_sigmoid(sum); + nnet[0].sprime[L][j] = nnet[0].svalue[L][j] * (1 - nnet[0].svalue[L][j]); + // note that d sigmoid / dz = sigmoid * (1 - sigmoid) + } + return TRUE; +} + +float nnet_sigmoid (float value) { + return 1.0 / (1.0 + exp(-value)); +} + +void nnet_sortseq (float *X, int *Y, int N) { + +# define SWAPFUNC(A,B){ float ftmp; int itmp; \ + ftmp = X[A]; X[A] = X[B]; X[B] = ftmp; \ + itmp = Y[A]; Y[A] = Y[B]; Y[B] = itmp; \ + } +# define COMPARE(A,B)(X[A] < X[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_train.save.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_train.save.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/nnet_train.save.c (revision 40477) @@ -0,0 +1,293 @@ +# include "data.h" +void sortseq (float *X, int *Y, int N); + +int nnet_train (int argc, char **argv) { + + int Nepoch = 10; + if ((N = get_argument (argc, argv, "-Nepoch"))) { + remove_argument (N, &argc, argv); + Nepoch = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + int Nmini = 100; + if ((N = get_argument (argc, argv, "-Nmini"))) { + remove_argument (N, &argc, argv); + Nmini = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc < 4) { + gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n"); + gprint (GP_ERR, "OPTIONS: -Nepoch [N] -Nmini [N]\n"); + return FALSE; + } + + Nnet *nnet = FindNnet (argv[1]); + if (nnet == NULL) { + gprint (GP_ERR, "nnet %s not found, create it first\n", argv[1]); + return FALSE; + } + + // save the number of input and output nodes; these need to match the supplied vectors + int Nlayer = nnet[0].Nlayer; + int Ninput = nnet[0].Nnodes[0]; + int Noutput = nnet[0].Nnodes[Nlayer - 1]; // number of input + output nodes + + // we need to have the right number of input and output vectors + if (argc != Ninput + Noutput + 2) { + gprint (GP_ERR, "need %d input and %d output vectors, but we have %d total\n", nnet[0].Nnodes[0], nnet[0].Nnodes[Nlayer - 1], argc - 2); + return FALSE; + } + + // grab the input and output vectors from the argument list + ALLOCATE_PTR (inVec, Vector *, Ninput); + ALLOCATE_PTR (outVec, Vector *, Noutput); + + // check that the vectors exist and that lengths match + int Ntrial = 0; + for (int i = 0; i < Ninput; i++) { + if ((inVec[i] = SelectVector (argv[i + 2], OLDVECTOR, FALSE)) == NULL) { + gprint (GP_ERR, "unknown input vector %s\n", argv[i+2]); + gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n"); + free (inVec); + free (outVec); + return (FALSE); + } + if (Ntrial && (inVec[i][0].Nelements != Ntrial)) { + gprint (GP_ERR, "input vectors have inconsistent lengths: %d vs %d for %s\n", Ntrial, inVec[i][0].Nelements, inVec[i][0].name); + free (inVec); + free (outVec); + return (FALSE); + } + if (!Ntrial) Ntrial = inVec[i][0].Nelements; + if (!Ntrial) { + gprint (GP_ERR, "trial vectors must be non-zero length\n"); + free (inVec); + free (outVec); + return (FALSE); + } + } + for (int i = 0; i < Noutput; i++) { + if ((outVec[i] = SelectVector (argv[i + 2 + Ninput], OLDVECTOR, FALSE)) == NULL) { + gprint (GP_ERR, "unknown output vector %s\n", argv[i+2+Ninput]); + gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n"); + free (inVec); + free (outVec); + return (FALSE); + } + if (outVec[i][0].Nelements != Ntrial) { + gprint (GP_ERR, "output and input vectors have inconsistent lengths: %d vs %d for %s\n", Ntrial, outVec[i][0].Nelements, outVec[i][0].name); + free (inVec); + free (outVec); + return (FALSE); + } + } + + // core of the training process: + + // index to hold a random sequence + ALLOCATE_PTR (seq, int, Ntrial); + ALLOCATE_PTR (rnd, float, Ntrial); + + // train for Nepochs + // this recreates 'SGD' from http://neuralnetworksanddeeplearning.com/chap1.html + for (int epoch = 0; epoch < Nepoch; epoch ++) { + + // generate a random sequence : used to select random mini batches + for (int i = 0; i < Ntrial; i++) { seq[i] = i; rnd[i] = drand48(); } + sortseq (rnd, seq, Ntrial); + + int Npass = Ntrial / Nmini; + for (int pass = 0; pass < Npass; pass ++) { + // for a given pass select seq elements pass*Nmini to pass*Nmini + Nmini - 1 + // update the weights and biases using the mini batch subset + nnet_descent_step (nnet, inVec, outVec, seq, pass, Nmini, eta); + gprint (GP_ERR, "epoch %d of %d, pass %d of %d\n", epoch, Nepoch, pass, Npass); + } + } + + free (seq); + free (rnd); + free (inVec); + free (outVec); + return TRUE; +} + +// this recreates 'update_mini_batch' from http://neuralnetworksanddeeplearning.com/chap1.html +void nnet_descent_step (NNet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta) { + + int Ntrial = inVec[0][0].Nelements; + + nnet_reset_Nabla (nnet); + + for (i = 0; (i < Nmini) && (pass*Nmini + i < Ntrial); i++) { + + // N is the element of the mini batch on which we are currently operating + N = seq[pass*Nmini + i]; + + // backprop generates a dNabla_b, dNabla_w pair for the element N of the input and output vectors + nnet_backprop (nnet, inVec, outVec, N); + + nnet_update_Nabla (nnet); + } + + nnet_apply_Nabla (nnet, Nmini, eta); +} + +void nnet_backprop (NNet *nnet, Vector **inVec, Vector **outVec, int N) { + + // start with the input values + + // store the input values for this row (trial) in the input value vector + int Nlayer = nnet[0].Nlayer; + int Ninput = nnet[0].Nnodes[0]; + int Noutput = nnet[0].Nnodes[Nlayer - 1]; // number of input + output nodes + + for (int j = 0; j < Ninput; j++) { + nnet[0].svalue[0][j] = inVec[j][0].elements.Flt[N]; + } + + // z = w * activation + bias + // save z [one per non-input layer] + // activation = sigmoid (z) + + nnet_feedforward (nnet); + + // backward pass (note for now these are vector operations: + for (L = Nlayer - 1; L > 0; L--) { + + // sp is array of same size as zvalue for each layer + // sp = sigmoid_prime (nnet[0].zvalue[L]); sprime is precalculated in feedforward + + if (L == Nlayer - 1) { + // starting point uses cost_derivative to compare last svalue set with truth output + // delta = cost_derivative(nnet[0].svalue[L], outVec, N) * sprime[L]; + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + nnet[0].delta[L][j] = cost_derivative(nnet[0].svalue[L][j], outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j]; + } + } else { + // delta = (delta DOT transpose(weights[L+1])) * sp; + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + tmpdelta[L][j] = 0; + for (int i = 0; i < nnet[0].Nnodes[L+1]; i++) { + int k = j + i*nnet[0].Nnodes[L]; + tmpdelta[L][j] += nnet[0].weight[L][k] * nnet[0].sprime[L][i]; // XXX check on the index values + } + tmpdelta[L][j] * nnet[0].sprime[L][j]; + } + } + + // UPDATE + Nabla_b[L] = delta; + + // Nabla_w[L] = delta DOT transpose(nnet[0].svalue[L-1]); + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) { + int k = i + j*nnet[0].Nnodes[n-1]; + nnet[0]. Nabla_w[n][k] = nnet[0].svalue[L-1][i] * nnet[0].delta[L][j]; + } + } + + } +} + +// support functions to loop over the Nabla entries +void nnet_apply_Nabla (NNet *nnet, int Nmini, float eta) { + for (int n = 1; n < nnet[0].Nlayer; n++) { + + for (int j = 0; j < nnet[0].Nnodes[n]; j++) { + nnet[0].biases[n][j] -= (eta / Nmini) * nnet[0].Nabla_b[n][j]; + + for (int i = 0; i < nnet[0].Nnodes[n-1]; i++) { + int k = i + j*nnet[0].Nnodes[n-1]; + nnet[0].weights[n][k] -= (eta / Nmini) * nnet[0].Nabla_w[n][k]; + } + } + } +} +void nnet_update_Nabla (NNet *nnet) { + for (int n = 1; n < nnet[0].Nlayer; n++) { + + for (int j = 0; j < nnet[0].Nnodes[n]; j++) { + nnet[0]. Nabla_b[n][j] += nnet[0].dNabla_b[n][j]; + + for (int i = 0; i < nnet[0].Nnodes[n-1]; i++) { + int k = i + j*nnet[0].Nnodes[n-1]; + nnet[0]. Nabla_w[n][k] += nnet[0].dNabla_w[n][k]; + } + } + } +} +void nnet_reset_Nabla (NNet *nnet) { + for (int n = 1; n < nnet[0].Nlayer; n++) { + + for (int j = 0; j < nnet[0].Nnodes[n]; j++) { + + nnet[0]. Nabla_b[n][j] = 0; + nnet[0].dNabla_b[n][j] = 0; + + for (int i = 0; i < nnet[0].Nnodes[n-1]; i++) { + int k = i + j*nnet[0].Nnodes[n-1]; + nnet[0]. Nabla_w[n][k] = 0; + nnet[0].dNabla_w[n][k] = 0; + } + } + } +} + +// the input values must already be copied to the input layer svalue[] +void nnet_feedforward (NNet *nnet) { + + for (int i = 1; i < nnet[0].Nlayer; i++) { + nnet_onelayer (nnet, i); + } + return; +} + +// calcularte z, sigmoid(z) for each layer (z = w*value + bias) +int nnet_onelayer (NNet *nnet, int n) { + + if (n < 1) return FALSE; // abort here? + if (n >= nnet[0].Nlayer) return FALSE; // abort here? + + // evaluating a single layer [n], n > 0, n < Nlayer: + int Ninput = nnet[0].Nnodes[n - 1]; + int Noutput = nnet[0].Nnodes[n]; + + // input layer is [n-1], output layer is [n] + for (int j = 0; j < Noutput; j ++) { + float sum = 0; + for (int i = 0; i < Ninput; i++) { + // weight matrix order is (0, 1, ... Ninput-1, Ninput, Ninput + 1, ... Ninput * Noutput - 1) + int k = j * Ninput + i; + sum += nnet[0].weight[n][k]*nnet[0].svalue[n-1][i]; + } + sum += nnet[0].biases[n][j]; + nnet[0].zvalue[n][j] = sum; + nnet[0].svalue[n][j] = nnet_sigmoid(sum); + nnet[0].sprime[n][j] = nnet[0].svalue[n][j] * (1 - nnet[0].svalue[n][j]); + // note that d sigmoid / dz = sigmoid * (1 - sigmoid) + } + return TRUE; +} + +float nnet_sigmoid (float value) { + return 1.0 / (1.0 + exp(-sum)); +} + +void sortseq (float *X, int *Y, int N) { + +# define SWAPFUNC(A,B){ float ftmp; int itmp; \ + ftmp = X[A]; X[A] = X[B]; X[B] = ftmp; \ + itmp = Y[A]; Y[A] = Y[B]; Y[B] = itmp; \ +} +# define COMPARE(A,B)(X[A] < X[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/periodogram-fm.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/periodogram-fm.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/periodogram-fm.c (revision 40477) @@ -0,0 +1,179 @@ +# include "data.h" +# define MIN_VAR 1e-8 +int periodogram_fm (int argc, char **argv) { + + int N; + + int VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + int OPTIMAL = FALSE; + if ((N = get_argument (argc, argv, "-optimal"))) { + OPTIMAL = TRUE; + remove_argument (N, &argc, argv); + } + + int LINEAR = FALSE; + if ((N = get_argument (argc, argv, "-linear"))) { + LINEAR = TRUE; + remove_argument (N, &argc, argv); + } + + int Nperiods = 1024; + if ((N = get_argument (argc, argv, "-Nperiods"))) { + remove_argument (N, &argc, argv); + Nperiods = atoi(argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc != 8) { + gprint (GP_ERR, "USAGE: periodogram_fm (time) (flux) (dflux) (minP) (maxP) (period) (power)\n"); + return (FALSE); + } + + // XXX allow dflux to be dropped? + + Vector *time, *flux, *dflux, *power, *period; + if ((time = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((flux = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((dflux = SelectVector (argv[3], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((period = SelectVector (argv[6], ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((power = SelectVector (argv[7], ANYVECTOR, TRUE)) == NULL) return (FALSE); + double minP = atof(argv[4]); + double maxP = atof(argv[5]); + + REQUIRE_VECTOR_FLT (time, FALSE); + REQUIRE_VECTOR_FLT (flux, FALSE); + REQUIRE_VECTOR_FLT (dflux, FALSE); + + /* find the max baseline, sum the inverse variances */ + double minT = time[0].elements.Flt[0]; + double maxT = time[0].elements.Flt[0]; + opihi_flt Weight = 0; + int Npt = time[0].Nelements; + + opihi_flt *tv = time[0].elements.Flt; + opihi_flt *df = dflux[0].elements.Flt; + for (int i = 0; i < Npt; i++, tv++, df++) { + minT = MIN (minT, *tv); + maxT = MAX (maxT, *tv); + // skip points with 0.0 error? or add minimum variance? + Weight += 1.0 / (SQ(*df) + MIN_VAR); // MIN_VAR : added in quadrature to avoid Inf (XXX make this a user parameter?) + } + + double WeightInv = 1.0 / Weight; + double dTime = maxT - minT; + if (dTime == 0) { + gprint (GP_ERR, "ERROR: time range is zero\n"); + return (FALSE); + } + + int Np = 0; + int NP = Nperiods; + ResetVector (power, OPIHI_FLT, NP); + ResetVector (period, OPIHI_FLT, NP); + + // for testing, we are going to write out the terms explicitly + // for optimization, some of the trig functions can be calculated more efficiently + // by storing cos,sin(w t) and using some recurrence rules + + // maxP = minP * dP^n [n = 0 -- Nperiods-1] + // log(maxP) = log(minP) + (Nperiods - 1) * log (dP) + // log(dP) = (log(maxP) - log(minP)) / (Nperiods - 1) + double dP = LINEAR ? (maxP - minP)/(Nperiods - 1) : pow(10.0, ((log10(maxP) - log10(minP))/(Nperiods - 1))); + + // XXX consider choice of period or frequency step + double P = minP; + for (Np = 0; (Np < Nperiods) && (P <= maxP); Np++) { + double w = 2*M_PI/P; + + /* find the period offset tau */ + tv = time[0].elements.Flt; + df = dflux[0].elements.Flt; + + double cs = 0.0, sn = 0.0, sn2 = 0.0, cs2 = 0.0; + + for (int i = 0; i < Npt; i++, tv++, df++) { + opihi_flt wt = WeightInv / (SQ(*df) + MIN_VAR); + cs += wt*cos (*tv*w); + sn += wt*sin (*tv*w); + cs2 += wt*cos (*tv*w*2); + sn2 += wt*sin (*tv*w*2); + } + double ytan = sn2 - 2*cs*sn; + double xtan = cs2 - (SQ(cs) - SQ(sn)); + double tau = 0.5*atan2 (ytan, xtan) / w; + + /* find the power at this period */ + tv = time[0].elements.Flt; + df = dflux[0].elements.Flt; + opihi_flt *fv = flux[0].elements.Flt; + + // YY = YY_s - Y_s*Y_s + // CC = CC_s - C_s*C_s + // SS = SS_s - S_s*S_s + // YC = YC_s - Y_s*C_s + // YS = YS_s - Y_s*S_s + + double YY_s = 0.0, CC_s = 0.0, SS_s = 0.0, YC_s = 0.0, YS_s = 0.0; + double Y_s = 0.0, C_s = 0.0, S_s = 0.0; + for (int i = 0; i < Npt; i++, tv++, fv++, df++) { + opihi_flt wt = WeightInv / (SQ(*df) + MIN_VAR); + cs = cos (w*(*tv-tau)); + sn = sin (w*(*tv-tau)); + + double wtf = *fv*wt; + Y_s += wtf; + C_s += wt*cs; + S_s += wt*sn; + + YY_s += wtf*(*fv); + YC_s += wtf*cs; + YS_s += wtf*sn; + + CC_s += wt*cs*cs; + SS_s += wt*sn*sn; + } + double YY = YY_s - SQ(Y_s); + double YC = YC_s - Y_s*C_s; + double YS = YS_s - Y_s*S_s; + double CC = CC_s - SQ(C_s); + double SS = SS_s - SQ(S_s); + + double Pa = SQ(YC) / CC; + double Pb = SQ(YS) / SS; + double Po = (Pa + Pb) / YY; + + power[0].elements.Flt[Np] = Po; + period[0].elements.Flt[Np] = P; + if (Np >= NP) { + NP += 100; + REALLOCATE (power[0].elements.Flt, opihi_flt, NP); + REALLOCATE (period[0].elements.Flt, opihi_flt, NP); + } + + // double ratio = 1 + 0.1*P/dTime; + + if (VERBOSE) gprint (GP_ERR, "tau: %f, P: %f, ratio: %f, dTime: %f, nextP: %f\n", tau, P, dP, dTime, P*dP); + + if (OPTIMAL) { + dP = 0.4 * SQ(P) / dTime; + P += dP; + continue; + } + + if (LINEAR) { + P += dP; + } else { + P *= dP; + } + } + + ResetVector (power, OPIHI_FLT, Np); + ResetVector (period, OPIHI_FLT, Np); + + return (TRUE); +} Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/print_vectors.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/print_vectors.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/print_vectors.c (revision 40477) @@ -4,5 +4,19 @@ Vector **vec; - int i, j; + int i, j, N; + + int START_VALUE = 0; + if ((N = get_argument (argc, argv, "-s"))) { + remove_argument (N, &argc, argv); + START_VALUE = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + int END_VALUE = -1; + if ((N = get_argument (argc, argv, "-e"))) { + remove_argument (N, &argc, argv); + END_VALUE = atoi (argv[N]); + remove_argument (N, &argc, argv); + } if (argc < 2) { @@ -27,5 +41,13 @@ } - for (j = 0; j < MaxLen; j++) { + // start and end may be 0 - N (truncated to N) or may be negative, in which case it refers to + // distance from the end (just like vector[-5]) + START_VALUE = (START_VALUE < 0) ? MaxLen + START_VALUE + 1 : MIN (START_VALUE, MaxLen); + START_VALUE = MAX (0, START_VALUE); + + END_VALUE = (END_VALUE < 0) ? MaxLen + END_VALUE + 1 : MIN (END_VALUE, MaxLen); + END_VALUE = MAX (0, END_VALUE); + + for (j = START_VALUE; j < END_VALUE; j++) { for (i = 0; i < Nvec; i++) { if (j >= vec[i][0].Nelements) { @@ -35,5 +57,5 @@ gprint (GP_LOG, "%f ", vec[i][0].elements.Flt[j]); } else { - gprint (GP_LOG, "%d ", vec[i][0].elements.Int[j]); + gprint (GP_LOG, OPIHI_INT_FMT" ", vec[i][0].elements.Int[j]); } } Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/read_vectors.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/read_vectors.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/read_vectors.c (revision 40477) @@ -19,9 +19,14 @@ } + if (strlen(argv[1]) >= 2048) { + gprint (GP_ERR, "filename %s is too long\n", argv[1]); + return (FALSE); + } + strcpy (filename, argv[1]); if (f != (FILE *) NULL) { fclose (f); } f = fopen (filename, "r"); if (f == (FILE *) NULL) { - gprint (GP_ERR, "failed to open file %s\n", argv[1]); + gprint (GP_ERR, "failed to open file %s\n", filename); return (FALSE); } Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/reindex.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/reindex.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/reindex.c (revision 40477) @@ -47,5 +47,5 @@ continue; } - if (*vx > Nmax) ESCAPE("unexpected value in index: %d (%d)\n", *vx, i); + if (*vx > Nmax) ESCAPE("unexpected value in index: "OPIHI_INT_FMT" (%d)\n", *vx, i); ovec[0].elements.Flt[Npts] = vi[*vx]; Npts++; @@ -67,5 +67,5 @@ continue; } - if (*vx > Nmax) ESCAPE("unexpected value in index: %d (%d)\n", *vx, i); + if (*vx > Nmax) ESCAPE("unexpected value in index: "OPIHI_INT_FMT" (%d)\n", *vx, i); ovec[0].elements.Int[Npts] = vi[*vx]; Npts++; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/sort.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/sort.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/sort.c (revision 40477) @@ -7,5 +7,7 @@ itmp = IDX[A]; IDX[A] = IDX[B]; IDX[B] = itmp; \ } -# define COMPARE(A,B)(X[A] < X[B]) + +// # define COMPARE(A,B)(X[A] < X[B]) +# define COMPARE(A,B)((!isfinite(X[A]) && isfinite(X[B])) || (X[A] < X[B])) OHANA_SORT (N, COMPARE, SWAPFUNC); @@ -22,5 +24,7 @@ itmp = IDX[A]; IDX[A] = IDX[B]; IDX[B] = itmp; \ } + # define COMPARE(A,B)(X[A] < X[B]) +// # define COMPARE(A,B)((!isfinite(X[A]) && isfinite(X[B])) || (X[A] < X[B])) OHANA_SORT (N, COMPARE, SWAPFUNC); Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/mslice.sh =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/mslice.sh (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/mslice.sh (revision 40477) @@ -0,0 +1,49 @@ + +macro test_00 + + mcreate t0 30 50 -nz 70 + + set t1 = xramp(t0) + set t2 = yramp(t0) + set t3 = zramp(t0) + + set r0 = sqrt((t1 - 15)^2 + (t2 - 25)^2 + (t3 - 35)^2) + + mslice r0 rz 35 -z + mslice r0 ry 25 -y + mslice r0 rx 15 -x + + mcreate Rz 30 50 + set Rz = sqrt((xramp(Rz) - 15)^2 + (yramp(Rz) - 25)^2 + (35 - 35)^2) + set dRz = rz - Rz + + mcreate Ry 30 70 + set Ry = sqrt((xramp(Ry) - 15)^2 + (25 - 25)^2 + (yramp(Ry) - 35)^2) + set dRy = ry - Ry + + mcreate Rx 50 70 + set Rx = sqrt((15 - 15)^2 + (xramp(Rx) - 25)^2 + (yramp(Rx) - 35)^2) + set dRx = rx - Rx + + tv -ch 1 dRx -5 10 + tv -ch 2 dRy -5 10 + tv -ch 3 dRz -5 10 + + mslice r0 rz 13 -z + mslice r0 ry 43 -y + mslice r0 rx 8 -x + + set Rz = sqrt((xramp(Rz) - 15)^2 + (yramp(Rz) - 25)^2 + (13 - 35)^2) + set dRz = rz - Rz + + set Ry = sqrt((xramp(Ry) - 15)^2 + (43 - 25)^2 + (yramp(Ry) - 35)^2) + set dRy = ry - Ry + + set Rx = sqrt((8 - 15)^2 + (xramp(Rx) - 25)^2 + (yramp(Rx) - 35)^2) + set dRx = rx - Rx + + stat dRx + stat dRy + stat dRz +end + Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/nnet.sh =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/nnet.sh (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/nnet.sh (revision 40477) @@ -0,0 +1,683 @@ + +macro test.io + + nnet create t1 2 3 2 + + # biases: + vlist v1 -0.2 0.0 0.2 + vlist v2 -0.3 0.3 + + # weights + mcreate m1 2 3 + m1[0][0] = 0.2 + m1[1][0] = -0.2 + m1[0][1] = -0.6 + m1[1][1] = 0.5 + m1[0][2] = -0.2 + m1[1][2] = 0.2 + + mcreate m2 3 2 + m2[0][0] = -0.6 + m2[1][0] = -0.4 + m2[2][0] = -0.5 + + m2[0][1] = -0.2 + m2[1][1] = -0.5 + m2[2][1] = 0.1 + + # set weights and biases + nnet set t1 m1 v1 m2 v2 + + nnet write t1 test.nnet.dat + nnet read t2 test.nnet.dat + + nnet get t2 M1 V1 M2 V2 + + set dM1 = M1 - m1 + set dM2 = M2 - m2 + set dV1 = V1 - v1 + set dV2 = V2 - v2 + + stat dM1 + stat dM2 + + vstat dV1 + vstat dV2 +end + +macro test.python + if ($0 != 4) + echo "USAGE: test.python (Nmini) (Nepoch) (eta)" + break + end + + # test I/O + + create x 0 100 + set y = dsin(x/10) + write test.dat x y + break + + local Nmini Nepoch eta + $Nmini = $1 + $Nepoch = $2 + $eta = $3 + + # create a nnet with just 2 inputs and 2 outputs + nnet create t1 2 2 + + # biases: + vlist v1 0.5 0.0 + + # weights + mcreate m1 2 2 + m1[0][0] = 0.5 + m1[1][0] = -0.2 + m1[0][1] = -0.5 + m1[1][1] = 0.2 + + # set weights and biases + nnet set t1 m1 v1 + + # generate an input set spanning the range -5 to +5 + + create n 0 1000 + set x0 = 10*rnd(n) - 5.0 + set x1 = 10*rnd(n) - 5.0 + + # get output vectors the these input vectors + nnet apply t1 x0 x1 y0 y1 + +# -large-weight-initializer : revert to original implementation +# nnet create tn 2 2 + nnet create tn 2 2 -large-weight-initializer + +# -quadratic-cost : revert to original implementation + nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 -quadratic-cost -resid resid -result result +# nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 +# nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 2.0 + + nnet apply tn x0 x1 Y0 Y1 + + set dy0 = y0 - Y0 + set dy1 = y1 - Y1 + + vstat dy0 + $yp = $MAX + $ym = $MIN + + vstat dy1 + $yp = max ($MAX , $yp) + $ym = min ($MIN , $ym) + $yp = 0.02; $ym = -0.02 + + style -pt circle -sz 2; lim x0 $ym $yp; clear; box; plot -c blue x0 dy0; plot -c red x0 dy1 +end + +macro test + + nnet create t0 5 10 15 3 + nnet list + + nnet create t1 6 4 5 + nnet list + + nnet create t1 5 8 + nnet list + + # vlist v0 1 2 3 4 5 + vlist v1 1 2 3 4 5 6 7 8 + + mcreate m1 5 8 + + # generate a weight vector of length v1[] + vlist my 9 8 7 6 5 4 3 2 + + for i 0 5 + mset m1 my -y $i + set my = my + $i + end + + # set weights and biases + nnet set t1 m1 v1 + + # get weights and biases + nnet get t1 M1 V1 + + # compare + set dv = v1 - V1 + vstat dv + + set dm = m1 - M1 + stat dm + +end + +macro test2 + + nnet create t1 2 2 + nnet list + + # output = sigmoid (sum(weight[i][j] * input[i]) + bias[j]) + + # biases of + vlist v1 0.8 0.2 + + mcreate m1 2 2 + m1[0][0] = 0.5 + m1[1][0] = 0.1 + m1[0][1] = -0.2 + m1[1][1] = 0.8 + + # set weights and biases + nnet set t1 m1 v1 + + vlist vin0 1 2 + vlist vin1 2 1 + + # get weights and biases + nnet apply t1 vin0 vin1 vout0 vout1 + + vlist z0 {m1[0][0]*vin0[0] + m1[1][0]*vin1[0] + v1[0]} {m1[0][0]*vin0[1] + m1[1][0]*vin1[1] + v1[0]} + vlist z1 {m1[0][1]*vin0[0] + m1[1][1]*vin1[0] + v1[1]} {m1[0][1]*vin0[1] + m1[1][1]*vin1[1] + v1[1]} + set t0 = 1 / (1 + exp(-1*z0)) + set t1 = 1 / (1 + exp(-1*z1)) + + # compare + set dt0 = t0 - vout0 + vstat dt0 + + set dt1 = t1 - vout1 + vstat dt1 +end + +macro test3 + + memory all + + nnet create t0 5 10 15 3 + nnet list + + nnet create t1 5 8 + nnet list + + # vlist v0 1 2 3 4 5 + vlist v1 1 2 3 4 5 6 7 8 + + mcreate m1 5 8 + + # generate a weight vector of length v1[] + vlist my 9 8 7 6 5 4 3 2 + + for i 0 5 + mset m1 my -y $i + set my = my + $i + end + + # set weights and biases + nnet set t1 m1 v1 + + # get weights and biases + nnet get t1 M1 V1 + + # compare + set dv = v1 - V1 + vstat dv + + set dm = m1 - M1 + stat dm + + nnet delete t0 + nnet delete t1 + + delete m1 M1 dm + delete v1 V1 dv my + + vectors + buffers + + memory all +end + +macro test.simple + if ($0 != 4) + echo "USAGE: test.simple (Nmini) (Nepoch) (eta)" + break + end + + local Nmini Nepoch eta + $Nmini = $1 + $Nepoch = $2 + $eta = $3 + + # create a nnet with just 2 inputs and 2 outputs + nnet create t1 2 2 + + # biases: + vlist v1 0.5 0.0 + + # weights + mcreate m1 2 2 + m1[0][0] = 0.5 + m1[1][0] = -0.2 + m1[0][1] = -0.5 + m1[1][1] = 0.2 + + # set weights and biases + nnet set t1 m1 v1 + + # generate an input set spanning the range -5 to +5 + + create n 0 1000 + set x0 = 10*rnd(n) - 5.0 + set x1 = 10*rnd(n) - 5.0 + + # get output vectors the these input vectors + nnet apply t1 x0 x1 y0 y1 + +# -large-weight-initializer : revert to original implementation +# nnet create tn 2 2 + nnet create tn 2 2 -large-weight-initializer + +# -quadratic-cost : revert to original implementation + nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 -quadratic-cost -resid resid -result result +# nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 +# nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 2.0 + + nnet apply tn x0 x1 Y0 Y1 + + set dy0 = y0 - Y0 + set dy1 = y1 - Y1 + + vstat dy0 + $yp = $MAX + $ym = $MIN + + vstat dy1 + $yp = max ($MAX , $yp) + $ym = min ($MIN , $ym) + $yp = 0.02; $ym = -0.02 + + style -pt circle -sz 2; lim x0 $ym $yp; clear; box; plot -c blue x0 dy0; plot -c red x0 dy1 +end + +macro test.bilevel + if ($0 != 4) + echo "USAGE: test.bilevel (Nmini) (Nepoch) (eta)" + break + end + + local Nmini Nepoch eta + $Nmini = $1 + $Nepoch = $2 + $eta = $3 + + # create a nnet with just 2 inputs and 2 outputs + nnet create t1 3 4 3 + + # biases: + vlist v1 -0.2 0.0 0.2 0.4 + vlist v2 -0.5 0.0 0.5 + + # weights + mcreate m1 3 4 + m1[0][0] = 0.2 + m1[1][0] = -0.2 + m1[2][0] = -0.3 + + m1[0][1] = -0.6 + m1[1][1] = 0.5 + m1[2][1] = -0.1 + + m1[0][2] = -0.2 + m1[1][2] = 0.2 + m1[2][2] = -0.5 + + m1[0][3] = 0.4 + m1[1][3] = -0.5 + m1[2][3] = -0.7 + + mcreate m2 4 3 + m2[0][0] = 0.8 + m2[1][0] = -0.4 + m2[2][0] = -0.5 + m2[3][0] = 0.3 + + m2[0][1] = -0.2 + m2[1][1] = -0.5 + m2[2][1] = 0.1 + m2[3][1] = 0.1 + + m2[0][2] = 0.2 + m2[1][2] = -0.2 + m2[2][2] = 0.5 + m2[3][2] = -0.1 + + # set weights and biases + nnet set t1 m1 v1 m2 v2 + + # generate an input set spanning the range -5 to +5 + + create n 0 10000 + set x0 = 10*rnd(n) - 5.0 + set x1 = 10*rnd(n) - 5.0 + set x2 = 10*rnd(n) - 5.0 + + # get output vectors the these input vectors + nnet apply t1 x0 x1 x2 y0 y1 y2 + +# nnet create tn 3 4 3 -large-weight-initializer + nnet create tn 3 4 3 + +# nnet train tn x0 x1 x2 y0 y1 y2 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.1 + nnet train tn x0 x1 x2 y0 y1 y2 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 -quadratic-cost -resid dS -result result + + nnet apply tn x0 x1 x2 Y0 Y1 Y2 + + set dy0 = y0 - Y0 + set dy1 = y1 - Y1 + set dy2 = y2 - Y2 + + vstat dy0 + $yp = $MAX + $ym = $MIN + + vstat dy1 + $yp = max ($MAX , $yp) + $ym = min ($MIN , $ym) + + vstat dy2 + $yp = max ($MAX , $yp) + $ym = min ($MIN , $ym) +# $yp = 0.02; $ym = -0.02 + + style -pt circle -sz 2; + lim x0 $ym $yp; clear; box; + plot -c blue x0 dy0; + plot -c red x0 dy1 + plot -c black x0 dy2 +end + +macro test.bilevel.small + if ($0 != 4) + echo "USAGE: test.bilevel (Nmini) (Nepoch) (eta)" + break + end + + local Nmini Nepoch eta + $Nmini = $1 + $Nepoch = $2 + $eta = $3 + + # create a nnet with just 2 inputs and 2 outputs + nnet create t1 2 3 2 + + # biases: + vlist v1 -0.2 0.0 0.2 + vlist v2 -0.3 0.3 + + # weights + mcreate m1 2 3 + m1[0][0] = 0.2 + m1[1][0] = -0.2 + m1[0][1] = -0.6 + m1[1][1] = 0.5 + m1[0][2] = -0.2 + m1[1][2] = 0.2 + + mcreate m2 3 2 + m2[0][0] = -0.6 + m2[1][0] = -0.4 + m2[2][0] = -0.5 + + m2[0][1] = -0.2 + m2[1][1] = -0.5 + m2[2][1] = 0.1 + + # set weights and biases + nnet set t1 m1 v1 m2 v2 + + # generate an input set spanning the range -5 to +5 + + create n 0 1000 + set x0 = 10*rnd(n) - 5.0 + set x1 = 10*rnd(n) - 5.0 + + # get output vectors the these input vectors + nnet apply t1 x0 x1 y0 y1 + +# nnet create tn 3 4 3 -large-weight-initializer + nnet create tn 2 3 2 + +# make the starting point close to the solution + m2[0][0] = -0.58 +# v2[1] = 0.25 + nnet set tn m1 v1 m2 v2 + +# nnet train tn x0 x1 x2 y0 y1 y2 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.1 +# nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 -quadratic-cost -resid dS -result result + nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 -resid dS -result result + + nnet apply tn x0 x1 Y0 Y1 + + set dy0 = y0 - Y0 + set dy1 = y1 - Y1 + + vstat dy0 + $yp = $MAX + $ym = $MIN + + vstat dy1 + $yp = max ($MAX , $yp) + $ym = min ($MIN , $ym) + + dev -n 0 + style -pt circle -sz 2; + lim x0 $ym $yp; clear; box; + plot -c blue x0 dy0; + plot -c red x0 dy1 + + set n = ramp(dS) + lim -n 1 n dS; clear; box; plot n dS +end + +macro test.bilevel.grid.wt + if ($0 != 4) + echo "USAGE: test.bilevel.grid.wt (level) (x) (y)" + break + end + + local Level ix iy + $Level = $1 + $ix = $2 + $iy = $3 + + # create a nnet with just 2 inputs and 2 outputs + nnet create t1 2 3 2 + + # biases: + vlist v1 -0.2 0.0 0.2 + vlist v2 -0.3 0.3 + + # weights + mcreate m1 2 3 + m1[0][0] = 0.2 + m1[1][0] = -0.2 + m1[0][1] = -0.6 + m1[1][1] = 0.5 + m1[0][2] = -0.2 + m1[1][2] = 0.2 + + mcreate m2 3 2 + m2[0][0] = -0.6 + m2[1][0] = -0.4 + m2[2][0] = -0.5 + + m2[0][1] = -0.2 + m2[1][1] = -0.5 + m2[2][1] = 0.1 + + # set weights and biases + nnet set t1 m1 v1 m2 v2 + + # generate an input set spanning the range -5 to +5 + + create n 0 1000 + set x0 = 10*rnd(n) - 5.0 + set x1 = 10*rnd(n) - 5.0 + + # get output vectors from these input vectors + nnet apply t1 x0 x1 y0 y1 + + # create a test nnet + nnet create tn 2 3 2 + + # 1D chi-square grid about truth + + set M1 = m1 + set M2 = m2 + set V1 = v1 + set V2 = v2 + + # disturb one element to see cross terms + M2[0][0] = -0.58 + + $To = M$Level[$ix][$iy] + + delete -q value svec0 svec1 sigvec + for frac 0.90 1.10 0.005 + M$Level[$ix][$iy] = $frac * $To + + nnet set tn M1 V1 M2 V2 + nnet apply tn x0 x1 Y0 Y1 + + set dy0 = y0 - Y0 + set dy1 = y1 - Y1 + + vstat -q dy0 + $S0 = $SIGMA + + vstat -q dy1 + $S1 = $SIGMA + + concat {$frac * $To} value + concat $S0 svec0 + concat $S1 svec1 + + concat {sqrt($S0^2 + $S1^2)} sigvec + end + + vstat -q sigvec + + lim -n 1 value -0.0001 $MAX; clear; box; + plot -c black value sigvec; + plot -c blue value svec0 -pt ocir -sz 2 ; + plot -c red value svec1 -pt ocir -sz 2 +end + +macro test.bilevel.grid.wt.2d + if ($0 != 7) + echo "USAGE: test.bilevel.grid.wt.2d (level) (x) (y) (level) (x) (y)" + break + end + + local LevelA ixA iyA LevelB ixB iyB + $LevelA = $1 + $ixA = $2 + $iyA = $3 + $LevelB = $4 + $ixB = $5 + $iyB = $6 + + # create a nnet with just 2 inputs and 2 outputs + nnet create t1 2 3 2 + + # biases: + vlist v1 -0.2 0.0 0.2 + vlist v2 -0.3 0.3 + + # weights + mcreate m1 2 3 + m1[0][0] = 0.2 + m1[1][0] = -0.2 + m1[0][1] = -0.6 + m1[1][1] = 0.5 + m1[0][2] = -0.2 + m1[1][2] = 0.2 + + mcreate m2 3 2 + m2[0][0] = -0.6 + m2[1][0] = -0.4 + m2[2][0] = -0.5 + + m2[0][1] = -0.2 + m2[1][1] = -0.5 + m2[2][1] = 0.1 + + # set weights and biases + nnet set t1 m1 v1 m2 v2 + + # generate an input set spanning the range -5 to +5 + + create n 0 1000 + set x0 = 10*rnd(n) - 5.0 + set x1 = 10*rnd(n) - 5.0 + + # get output vectors from these input vectors + nnet apply t1 x0 x1 y0 y1 + + # create a test nnet + nnet create tn 2 3 2 + + # 1D chi-square grid about truth + + set M1 = m1 + set M2 = m2 + set V1 = v1 + set V2 = v2 + + # disturb one element to see cross terms + M2[0][0] = -0.58 + + $ToA = M$LevelA[$ixA][$iyA] + $ToB = M$LevelB[$ixB][$iyB] + + delete -q valueA valueB sigvec + + mcreate sigbuf 41 41 + + $ix = 0 + for fracA 0.90 1.10 0.005 + M$LevelA[$ixA][$iyA] = $fracA * $ToA + + $iy = 0 + for fracB 0.90 1.10 0.005 + + M$LevelB[$ixB][$iyB] = $fracB * $ToB + + nnet set tn M1 V1 M2 V2 + nnet apply tn x0 x1 Y0 Y1 + + set dy0 = y0 - Y0 + set dy1 = y1 - Y1 + + vstat -q dy0 + $S0 = $SIGMA + + vstat -q dy1 + $S1 = $SIGMA + + $sigval = sqrt($S0^2 + $S1^2) + concat {$fracA * $ToA} valueA + concat {$fracB * $ToB} valueB + concat $sigval sigvec + + sigbuf[$ix][$iy] = $sigval + $iy ++ + end + $ix ++ + end + + stat -q sigbuf + tv -n tv sigbuf $MIN {$MAX - $MIN} +end Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/periodogram-fm.sh =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/periodogram-fm.sh (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/periodogram-fm.sh (revision 40477) @@ -0,0 +1,589 @@ + +if (not($?PLOT)) set PLOT = 0 + +list tests + test1 + test2 + test3 + memtest1 +end + +# test using even samples +macro test1 + $PASS = 1 + break -auto off + + local P PI + $PI = 3.14159265359 + $P = 15.0 + + delete -q t f period power + + create t 0 100 + set f = sin(2*$PI*t/$P) + set df = 0.01 + zero(f) + + periodogram_fm t f df 5 50 period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.5) + $PASS = 0 + echo "OFFSET: {$peakpos - $P}" + end + + if ($PLOT) + lim period power; clear; box; line -c red70 -lw 3 $P 0 to $P $peakval; plot period power -x line + end +end + +# test using even samples +macro test1a + $PASS = 1 + break -auto off + + local P PI + $PI = 3.14159265359 + $P = 15.0 + + delete -q t f period power + + create t 0 100 + set f = sin(2*$PI*t/$P) + set df = 0.01 + zero(f) + + periodogram_fm t f df 5 50 period power -Nperiods 512 + + peak -q period power + + if (abs ($peakpos - $P) > 0.5) + $PASS = 0 + echo "OFFSET: {$peakpos - $P}" + end + + if ($PLOT) + lim period power; clear; box; line -c red70 -lw 3 $P 0 to $P $peakval; plot period power -x line + end +end + +# test using random samples +macro test2 + $PASS = 1 + break -auto off + + local P PI + $PI = 3.14159265359 + $P = 15.0 + + delete -q x t f period power + + create x 0 100 + set t = 100 * rnd(x) + set f = sin(2*$PI*t/$P) + set df = 0.01 + zero(f) + + periodogram_fm t f df 5 50 period power + + # lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f + # lim -n 1 period power; clear; box; plot period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.5) + $PASS = 0 + end + + if ($PLOT) + lim period power; clear; box; line -c red70 -lw 3 $P 0 to $P $peakval; plot period power -x line + end +end + +# test using random samples, higher frequency +macro test3 + $PASS = 1 + break -auto off + + local P PI + $PI = 3.14159265359 + $P = 2.0 + + delete -q x t f period power + + create x 0 100 + set t = 100 * rnd(x) + set f = sin(2*$PI*t/$P) + set df = 0.01 + zero(f) + + periodogram_fm t f df 1 10 period power + + # lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f + # lim -n 1 period power; clear; box; plot period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.05) + $PASS = 0 + end + + if ($PLOT) + lim period power; clear; box; line -c red70 -lw 3 $P 0 to $P $peakval; plot period power -x line + end +end + +# test using random samples, offset start +macro test4 + $PASS = 1 + break -auto off + + local P PI + $PI = 3.14159265359 + $P = 15.0 + + delete -q x t f period power + + create x 500 800 + set t = 300 * rnd(x) + 500 + set f = sin(2*$PI*t/$P) + set df = 0.01 + zero(f) + + periodogram_fm t f df 2 30 period power + +# lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f +# lim -n 1 period power; clear; box; plot period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.05) + $PASS = 0 + end + + if ($PLOT) + lim period power; clear; box; line -c red70 -lw 3 $P 0 to $P $peakval; plot period power -x line + end +end + +# test using random samples, offset start, non-zero DC +macro test5 + $PASS = 1 + break -auto off + + local P PI + $PI = 3.14159265359 + $P = 15.0 + + delete -q x t f period power + + create x 500 800 + set t = 300 * rnd(x) + 500 + set f = sin(2*$PI*t/$P) + 0.5 + set df = 0.01 + zero(f) + + periodogram_fm t f df 2 30 period power + +# lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f +# lim -n 1 period power; clear; box; plot period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.05) + $PASS = 0 + end + + if ($PLOT) + lim period power; clear; box; line -c red70 -lw 3 $P 0 to $P $peakval; plot period power -x line + end +end + +# test using 300 random samples, offset start, non-zero DC, some noise +macro test6 + $PASS = 1 + break -auto off + + local P PI + $PI = 3.14159265359 + $P = 15.0 + + delete -q x t f period power + + create x 500 800 + set t = 300 * rnd(x) + 500 + set fraw = sin(2*$PI*t/$P) + 0.5 + + # 0.05 : peakpos = 14.95 + # 0.10 : peakpos = 15.04 ( + gaussdev df t[] 0.0 0.25 + set f = fraw + df + set df = 0.01 + zero(f) + + periodogram_fm t f df 2 30 period power + +# lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f +# lim -n 1 period power; clear; box; plot period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.05) + $PASS = 0 + end + + if ($PLOT) + lim period power; clear; box; line -c red70 -lw 3 $P 0 to $P $peakval; plot period power -x line + end +end + +# test using 100 fewer random samples, offset start, non-zero DC, some noise +macro test7 + $PASS = 1 + break -auto off + + local P PI + $PI = 3.14159265359 + $P = 15.0 + + delete -q x t f period power + + create x 0 100 + set t = 100 * rnd(x) + set fraw = sin(2*$PI*t/$P) + 0.5 + + # 0.05 : peakpos = 14.95 + # 0.10 : peakpos = 15.04 ( + gaussdev df t[] 0.0 0.25 + set f = fraw + df + set df = 0.01 + zero(f) + + periodogram_fm t f df 2 30 period power + +# lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f +# lim -n 1 period power; clear; box; plot period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.05) + $PASS = 0 + end + + if ($PLOT) + lim period power; clear; box; line -c red70 -lw 3 $P 0 to $P $peakval; plot period power -x line + end +end + +# test using Ndays random samples, RR Lyrae-sized light curves (0.7 mag), +# optional noise level +macro test8 + if ($0 != 4) + echo "USAGE: test8: Period Ndays (df)" + break + end + + local Ndays + $P = $1 + $Ndays = $2 + $dM = $3 + + $PASS = 1 + break -auto off + + local PI + $PI = 3.14159265359 + $trueP = $P + + delete -q x t f period power + + create x 0 $Ndays + + # t is a time in days, but we always have 4 within 1 hour: + set tday = int(100 * rnd(x)); # choose Ndays random days between 0 and 100 + set dtx = (3/24) * rnd(x); # choose a starting time within that night + set t0 = tday + dtx + + set dt1 = (15.0 / 1440) * rnd(x) + ( 0 + 7.5) / 1440 + set dt2 = (15.0 / 1440) * rnd(x) + (15 + 7.5) / 1440 + set dt3 = (15.0 / 1440) * rnd(x) + (30 + 7.5) / 1440 + + delete -q t + concat t0 t + set tmp = t0 + dt1; concat tmp t + set tmp = t0 + dt2; concat tmp t + set tmp = t0 + dt3; concat tmp t + + set fraw = 0.75*sin(2*$PI*t/$P) + + # 0.05 : peakpos = 14.95 + # 0.10 : peakpos = 15.04 ( + gaussdev df t[] 0.0 $dM + set f = fraw + df + set df = $dM + zero(f) + + periodogram_fm t f df 0.1 20.0 period power + +# lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f +# lim -n 1 period power; clear; box; plot period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.05) + $PASS = 0 + end + if ($PLOT) + lim period power; clear; box; line -c red70 -lw 3 $P 0 to $P $peakval; plot period power -x line + end +end + +# test using Ndays random samples, RR Lyrae-sized light curves (0.7 mag), +# optional noise level +# compare periodogram and periodogram_fm +macro test9 + if ($0 != 4) + echo "USAGE: test8: Period Ndays (df)" + break + end + + local Ndays + $P = $1 + $Ndays = $2 + $dM = $3 + + $PASS = 1 + break -auto off + + local PI + $PI = 3.14159265359 + $trueP = $P + + delete -q x t f period power + + create x 0 $Ndays + + # t is a time in days, but we always have 4 within 1 hour: + set tday = int(100 * rnd(x)); # choose Ndays random days between 0 and 100 + set dtx = (3/24) * rnd(x); # choose a starting time within that night + set t0 = tday + dtx + + set dt1 = (15.0 / 1440) * rnd(x) + ( 0 + 7.5) / 1440 + set dt2 = (15.0 / 1440) * rnd(x) + (15 + 7.5) / 1440 + set dt3 = (15.0 / 1440) * rnd(x) + (30 + 7.5) / 1440 + + delete -q t + concat t0 t + set tmp = t0 + dt1; concat tmp t + set tmp = t0 + dt2; concat tmp t + set tmp = t0 + dt3; concat tmp t + + set fraw = 0.75*sin(2*$PI*t/$P) + + # 0.05 : peakpos = 14.95 + # 0.10 : peakpos = 15.04 ( + gaussdev df t[] 0.0 $dM + set f = fraw + df + set df = $dM + zero(f) + + periodogram_fm t f df 0.1 20.0 period_fm power_fm + periodogram t f 0.1 20.0 period power + +# lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f +# lim -n 1 period power; clear; box; plot period power + + peak -q period_fm power_fm + $peakval_fm = $peakval + + peak -q period power + vstat -q power + set power = power / $MAX + +# if (abs ($peakpos - $P) > 0.05) +# $PASS = 0 +# end + + set freq = 1 / period + set freq_fm = 1 / period_fm + $Freq = 1 / $P + + if ($PLOT) + if (1) + lim period power; clear; box + line -c red70 -lw 3 $P 0 to $P $peakval_fm; + plot period power -x line -c grey70 -lw 2 + plot period_fm power_fm -x line -c black + else + lim freq power; clear; box + line -c red70 -lw 3 $Freq 0 to $Freq 1.0 + plot freq power -x line -c grey70 -lw 2 + plot freq_fm power_fm -x line -c black + end + end +end + + +# test using Ndays random samples, RR Lyrae-sized light curves (0.7 mag), +# optional noise level +# compare periodogram and periodogram_fm +macro test10 + if ($0 != 4) + echo "USAGE: test8: Period Ndays (df)" + break + end + + local Ndays + $P = $1 + $Ndays = $2 + $dM = $3 + + $PASS = 1 + break -auto off + + local PI + $PI = 3.14159265359 + $trueP = $P + + delete -q x t f period power + + create x 0 $Ndays + + # t is a time in days, but we always have 4 within 1 hour: + set tday = int(100 * rnd(x)); # choose Ndays random days between 0 and 100 + set dtx = (3/24) * rnd(x); # choose a starting time within that night + set t0 = tday + dtx + + set dt1 = (15.0 / 1440) * rnd(x) + ( 0 + 7.5) / 1440 + set dt2 = (15.0 / 1440) * rnd(x) + (15 + 7.5) / 1440 + set dt3 = (15.0 / 1440) * rnd(x) + (30 + 7.5) / 1440 + + delete -q t + concat t0 t + set tmp = t0 + dt1; concat tmp t + set tmp = t0 + dt2; concat tmp t + set tmp = t0 + dt3; concat tmp t + + set fraw = 0.75*sin(2*$PI*t/$P) + + # 0.05 : peakpos = 14.95 + # 0.10 : peakpos = 15.04 ( + gaussdev df t[] 0.0 $dM + set f = fraw + df + set df = $dM + zero(f) + + periodogram_fm t f df 0.05 20.0 period_fm power_fm + + gaussdev df t[] 0.0 $dM + set Fo = df + periodogram_fm t Fo df 0.05 20.0 period power + +# lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f +# lim -n 1 period power; clear; box; plot period power + + peak -q period_fm power_fm + $peakval_fm = $peakval + + peak -q period power + +# if (abs ($peakpos - $P) > 0.05) +# $PASS = 0 +# end + + set freq = 1 / period + set freq_fm = 1 / period_fm + $Freq = 1 / $P + + if ($PLOT) + if (1) + lim period_fm power_fm; clear; box + line -c red70 -lw 3 $P 0 to $P $peakval_fm; + plot period power -x line -c grey70 -lw 2 + plot period_fm power_fm -x line -c black + else + lim freq power; clear; box + line -c red70 -lw 3 $Freq 0 to $Freq 1.0 + plot freq power -x line -c grey70 -lw 2 + plot freq_fm power_fm -x line -c black + end + end +end + +# we have time (MJD) and mag +# we generate the folded lightcure and measure sigma relative to the smoothed version (bins of 0.1 period) +macro fold.one.period + if ($0 != 5) + echo "USAGE: fold.one.period (time) (mag) (magErr) (period)" + break + end + + local myTime myMag myMagErr myPeriod + $myTime = $1 + $myMag = $2 + $myMagErr = $3 + $myPeriod = $4 + + set phi = $myTime / $myPeriod - int($myTime / $myPeriod) + + if ($PLOT_FOLD) + lim -n phi phi $myMag; clear; box; + end + + delete -q magResid + + $dPhi = 0.05; # half of bin size + create nphi $dPhi {1 + $dPhi} {2*$dPhi} + set magR = zero(nphi) + set magS = zero(nphi) + for i 0 nphi[] + subset tmp_mag_sub = $myMag where (phi >= nphi[$i] - $dPhi) && (phi < nphi[$i] + $dPhi) + vstat -q tmp_mag_sub + magR[$i] = $MEDIAN + magS[$i] = $SIGMA + + set magDelta = tmp_mag_sub - $MEDIAN + concat magDelta magResid + + if ($PLOT_FOLD) + subset tmp_phi_sub = phi where (phi >= nphi[$i] - $dPhi) && (phi < nphi[$i] + $dPhi) + if ($i % 2) + plot tmp_phi_sub tmp_mag_sub -pt 7 -sz 3 -c blue -lw 2 + else + plot tmp_phi_sub tmp_mag_sub -pt 7 -sz 3 -c red -lw 2 + end + end + end + + if ($PLOT_FOLD) + plot -pt 10 -sz 1.5 phi $myMag -dy $myMagErr + plot -pt 2 -sz 2.0 -c red nphi magR -dy magS + end + + vstat -q magResid +end + + + +# Memory test +macro memtest1 + + local i + local P PI + $PI = 3.14159265359 + $P = 15.0 + + delete -q x t f period power + + create x 500 800 + set t = 300 * rnd(x) + 500 + set f = sin(2*$PI*t/$P) + + list word -x "ps -p $PID -o rss" + $startmem = $word:1 + + for i 0 100 + periodogram_fm t f df 2 30 period power + end + + list word -x "ps -p $PID -o rss" + $endmem = $word:1 + + $PASS = 1 + + if ($endmem - $startmem > 180) + $PASS = 0 + echo "growth: {$endmem-$startmem}" + echo "kB/loop: {($endmem-$startmem)/100}" + end +end Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/periodogram.sh =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/periodogram.sh (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/periodogram.sh (revision 40477) @@ -1,2 +1,4 @@ + +if (not($?PLOT)) set PLOT = 0 list tests @@ -28,4 +30,8 @@ echo "OFFSET: {$peakpos - $P}" end + + if ($PLOT) + lim period power; clear; box; line -c red70 -lw 3 $P 0 to $P $peakval; plot period power -x line + end end @@ -107,4 +113,155 @@ if (abs ($peakpos - $P) > 0.05) $PASS = 0 + end +end + +# test using random samples, offset start, non-zero DC +macro test5 + $PASS = 1 + break -auto off + + local P PI + $PI = 3.14159265359 + $P = 15.0 + + delete -q x t f period power + + create x 500 800 + set t = 300 * rnd(x) + 500 + set f = sin(2*$PI*t/$P) + 0.5 + + periodogram t f 2 30 period power + +# lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f +# lim -n 1 period power; clear; box; plot period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.05) + $PASS = 0 + end +end + +# test using random samples, offset start, non-zero DC, some noise +macro test6 + $PASS = 1 + break -auto off + + local P PI + $PI = 3.14159265359 + $P = 15.0 + + delete -q x t f period power + + create x 500 800 + set t = 300 * rnd(x) + 500 + set fraw = sin(2*$PI*t/$P) + 0.5 + + # 0.05 : peakpos = 14.95 + # 0.10 : peakpos = 15.04 ( + gaussdev df t[] 0.0 0.25 + set f = fraw + df + + periodogram t f 2 30 period power + +# lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f +# lim -n 1 period power; clear; box; plot period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.05) + $PASS = 0 + end +end + +# test using fewer random samples, offset start, non-zero DC, some noise +macro test7 + $PASS = 1 + break -auto off + + local P PI + $PI = 3.14159265359 + $P = 15.0 + + delete -q x t f period power + + create x 0 100 + set t = 100 * rnd(x) + set fraw = sin(2*$PI*t/$P) + 0.5 + + # 0.05 : peakpos = 14.95 + # 0.10 : peakpos = 15.04 ( + gaussdev df t[] 0.0 0.25 + set f = fraw + df + + periodogram t f 2 30 period power + +# lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f +# lim -n 1 period power; clear; box; plot period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.05) + $PASS = 0 + end +end + +# test using fewer random samples, high frequency, non-zero DC, some noise +macro test8 + if ($0 != 4) + echo "USAGE: test8: Period Ndays df" + break + end + + local Ndays + $P = $1 + $Ndays = $2 + $dM = $3 + + $PASS = 1 + break -auto off + + local PI + $PI = 3.14159265359 + $trueP = $P + + delete -q x t f period power + + create x 0 $Ndays + + # t is a time in days, but we always have 4 within 1 hour: + set tday = int(100 * rnd(x)); # choose Ndays random days between 0 and 100 + set dtx = (3/24) * rnd(x); # choose a starting time within that night + set t0 = tday + dtx + + set dt1 = (15.0 / 1440) * rnd(x) + ( 0 + 7.5) / 1440 + set dt2 = (15.0 / 1440) * rnd(x) + (15 + 7.5) / 1440 + set dt3 = (15.0 / 1440) * rnd(x) + (30 + 7.5) / 1440 + + delete -q t + concat t0 t + set tmp = t0 + dt1; concat tmp t + set tmp = t0 + dt2; concat tmp t + set tmp = t0 + dt3; concat tmp t + + set fraw = 0.75*sin(2*$PI*t/$P) + + # 0.05 : peakpos = 14.95 + # 0.10 : peakpos = 15.04 ( + gaussdev df t[] 0.0 $dM + set f = fraw + df + + periodogram t f 0.1 2.0 period power + +# lim -n 0 t f; clear; box; plot -x 2 -pt 2 t f +# lim -n 1 period power; clear; box; plot period power + + peak -q period power + + if (abs ($peakpos - $P) > 0.05) + $PASS = 0 + end + if ($PLOT) + lim period power; clear; box; line -c red70 -lw 3 $P 0 to $P $peakval; plot period power -x line end end Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/uniqpair.sh =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/uniqpair.sh (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/uniqpair.sh (revision 40477) @@ -0,0 +1,83 @@ + + +list tests + test1 + memtest1 +end + +# Test if uniqpair works +macro test1 + + $PASS = 1 + + local i + + delete ID1 ID2 + + vlist -int ID1 1 1 2 3 3 4 5 6 + vlist -int ID2 2 3 4 5 5 6 7 8 + + uniqpair ID1 ID2 IDu -c IDn -d index + + # if ((xvec[1024] != 17) || (yvec[1024] != 100)) + # $PASS = 0 + # echo "Value mismatch: xvec[1024] yvec[1024] (should be 17,100)" + # end + # + # imhist -q buff xvec yvec -region 40 0 25 10 -range 0 10 + # + # if ((xvec[1024] != 10) || (yvec[1024] != 100)) + # $PASS = 0 + # echo "Value mismatch: xvec[1024] yvec[1024] (should be 10,100)" + # end + +end + +# Test if uniqpair works +macro test2 + + $PASS = 1 + + delete ID1 ID2 + + vlist -int ID1 {2^16 + 2} {2^17} {2^18 + 3} {2^16 + 2} {2^17 + 2} {2^18 + 3} + vlist -int ID2 {2^16 + 0} {2^17 + 2} {2^18 + 5} {2^16 + 0} {2^17 + 2} {2^18 + 5} + + uniqpair ID1 ID2 IDu -c IDn -d index + + reindex ID2s = ID2 using index + reindex ID1s = ID1 using index + + vectors + + echo "unique" + print_v IDu IDn + + echo "dups" + print_v ID1s ID2s index +end + + +# Memory test +macro memtest1 + + local i + + list word -x "ps -p $PID -o rss" + $startmem = $word:1 + + for i 0 1000 + imhist -q buff xvec yvec -region 40 0 25 10 -range 0 10 + end + + list word -x "ps -p $PID -o rss" + $endmem = $word:1 + + $PASS = 1 + + if ($endmem - $startmem > 10) + $PASS = 0 + echo "growth: {$endmem-$startmem}" + echo "kB/loop: {($endmem-$startmem)/1000}" + end +end Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/vlorentz.sh =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/vlorentz.sh (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/test/vlorentz.sh (revision 40477) @@ -0,0 +1,175 @@ + +list tests + test1 + test2 + test3 +end + +macro test1 + $PASS = 1 + break -auto off + + create x -10 10 0.1 + + $xo = 0.0 + $go = 2.0 + $Io = 10.0 + $so = 0.0 + + set ytru = $Io * $go / ((x - $xo)^2 + $go^2 / 4.0) / 2 / $M_PI + $so + gaussdev ystd ytru[] 0.0 1.0 + # set yobs = ytru + ystd * sqrt(abs(ytru)) + set yobs = ytru + ystd * 0.05 + + $C0 = 0.5 + $C1 = 3 + $C2 = 20 + $C3 = 0 + # set dy = sqrt(abs(yobs)) + set dy = 0.05 + zero(yobs) + + vlorentz x yobs dy yfit + lim x yobs; clear; box; plot -x 1 -c black x yobs; plot -c red x yfit + + if (abs($C0 - 0.0) > 0.01) + $PASS = 0 + end + if (abs($C1 - 3.0) > 0.01) + $PASS = 0 + end + if (abs($C2 - 5.0) > 0.01) + $PASS = 0 + end + if (abs($C3 - 0.0) > 0.01) + $PASS = 0 + end +end + +macro test1a + $PASS = 1 + break -auto off + + create x -10 10 0.1 + + $xo = 0.0 + $go = 2.0 + $Io = 10.0 + $so = 0.0 + + set ytru = $Io * $go / ((x - $xo)^2 + $go^2 / 4.0) / 2 / $M_PI + $so + gaussdev ystd ytru[] 0.0 1.0 + # set yobs = ytru + ystd * sqrt(abs(ytru)) + set yobs = ytru + ystd * 0.05 + set dy = 0.05 + zero(yobs) + + delete rC0 rC1 rC2 rC3 vC0 vC1 vC2 vC3 + + for i 0 500 + + $C0 = -1 + 2*rnd(0) + $C1 = 0.5 + 5*rnd(0) + $C2 = 2.0 + 30*rnd(0) + $C3 = -10 + 20*rnd(0) + + concat $C0 rC0 + concat $C1 rC1 + concat $C2 rC2 + concat $C3 rC3 + + vlorentz -q x yobs dy yfit + # lim x yobs; clear; box; plot -x 1 -c black x yobs; plot -c red x yfit + + concat $C0 vC0 + concat $C1 vC1 + concat $C2 vC2 + concat $C3 vC3 + end + + if (abs($C0 - 0.0) > 0.01) + $PASS = 0 + end + if (abs($C1 - 3.0) > 0.01) + $PASS = 0 + end + if (abs($C2 - 5.0) > 0.01) + $PASS = 0 + end + if (abs($C3 - 0.0) > 0.01) + $PASS = 0 + end +end + +# noise of 0.1 +macro test2 + $PASS = 1 + break -auto off + + create x -10 10 0.1 + set y = 1000 * exp(-0.5*x^2/3^2) + set dy = (rnd(y) - 0.5)/0.5 + set y = y + dy + + $C0 = 0.5 + $C1 = 2 + $C2 = 900 + $C3 = 1 + + vgauss -q x y dy yfit + # lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + if (abs($C0 - 0.0) > 0.01) + $PASS = 0 + end + if (abs($C1 - 3.0) > 0.01) + $PASS = 0 + end + if (abs($C2 - 1000.0) > 0.1) + $PASS = 0 + end + if (abs($C3 - 0.0) > 0.1) + $PASS = 0 + end +end + +# poisson-distributed noise +macro test3 + $PASS = 1 + break -auto off + + $C0o = 0.5 + $C1o = 2 + $C2o = 900 + $C3o = 1 + + create x -10 10 0.1 + set y = $C3o + $C2o * exp(-0.5*(x - $C0o)^2/$C1o^2) + + gaussdev dY y[] 0.0 1.0 + set dy = dY * sqrt(y) + set y = y + dy + + $C0 = $C0o + 2 + $C1 = $C1o + 2 + $C2 = $C2o + 50 + $C3 = $C3o + 10 + + vgauss -q x y dy yfit + # lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + $dS = 3.0 / sqrt(y[]) + + if (abs($C0 - $C0o) > $dS) + $PASS = 0 + end + if (abs($C1 - $C1o)/$C1o > $dS) + $PASS = 0 + end + if (abs($C2 - $C2o)/$C2o > $dS) + $PASS = 0 + end + if (abs($C3 - $C3o) > $dS) + $PASS = 0 + end +end Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/textline.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/textline.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/textline.c (revision 40477) @@ -6,8 +6,26 @@ char name[64]; double x, y, angle; - int kapa; Graphdata graphmode; - if (!style_args (&graphmode, &argc, argv, &kapa)) return (FALSE); + // if (!style_args (&graphmode, &argc, argv, &kapa)) return (FALSE); + + // Using only these style_args options + char *kapaName = NULL; + int kapa = -1; + if ((N = get_argument (argc, argv, "-n"))) { + remove_argument (N, &argc, argv); + kapaName = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!GetGraph (&graphmode, &kapa, kapaName)) return (FALSE); + FREE (kapaName); + + int color = KapaColorByName ("black"); + if ((N = get_argument (argc, argv, "-c"))) { + remove_argument (N, &argc, argv); + color = KapaColorByName (argv[N]); + if (color == -1) return (FALSE); + remove_argument (N, &argc, argv); + } if ((N = get_argument (argc, argv, "-fn"))) { @@ -34,6 +52,13 @@ } + int justify = 5; // default + if ((N = get_argument (argc, argv, "-justify"))) { + remove_argument (N, &argc, argv); + justify = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if (argc != 4) { - gprint (GP_ERR, "USAGE: text x y (line) [-fn (font) size] [-rot angle]\n"); + gprint (GP_ERR, "USAGE: text x y (line) [-fn (font) size] [-rot angle] [-justify N]\n"); return (FALSE); } @@ -52,5 +77,5 @@ } - KapaSendTextline (kapa, argv[3], x, y, angle); + KapaSendTextline (kapa, argv[3], x, y, angle, justify, color); return (TRUE); } Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/uniq.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/uniq.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/uniq.c (revision 40477) @@ -80,5 +80,5 @@ memcpy (indata, ivec->elements.Int, ivec[0].Nelements*sizeof(opihi_int)); - isort (indata, ivec->Nelements); + llsort (indata, ivec->Nelements); Nnew = 0; Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/uniqpair.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/uniqpair.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/uniqpair.c (revision 40477) @@ -0,0 +1,146 @@ +# include "data.h" + +/* given two 32bit IDs, a 64bit joint ID and sort it, along with an index (sequence) vector */ +int uniqpair (int argc, char **argv) { + + int N; + Vector *ID1vec = NULL, *ID2vec = NULL, *OUTvec = NULL; + + int VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + remove_argument (N, &argc, argv); + VERBOSE = TRUE; + } + + Vector *SEQdup = NULL; + if ((N = get_argument (argc, argv, "-d"))) { + remove_argument (N, &argc, argv); + if ((SEQdup = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) { + gprint (GP_ERR, "invalid vector %s\n", "SEQdups"); + return FALSE; + } + remove_argument (N, &argc, argv); + } + + Vector *CNTvec = NULL; + if ((N = get_argument (argc, argv, "-c"))) { + remove_argument (N, &argc, argv); + if ((CNTvec = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) { + gprint (GP_ERR, "invalid vector %s\n", argv[N]); + return FALSE; + } + remove_argument (N, &argc, argv); + } + + if (argc != 4) { + gprint (GP_ERR, "USAGE: uniqpair (ID1) (ID2) (IDout) [-c count] [-d dupindex]\n"); + gprint (GP_ERR, " merge ID1 and ID2 (32 bit int values) into a single index and find unique entries\n"); + gprint (GP_ERR, " -c count: save the number of each unique entry in the count vector\n"); + gprint (GP_ERR, " -d dupindex: save the sequence number of duplicate entries in the dupindex vector\n"); + return (FALSE); + } + + if ((ID1vec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((ID2vec = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((OUTvec = SelectVector (argv[3], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + if ((ID1vec->type == OPIHI_FLT) || (ID2vec->type == OPIHI_FLT)) { + gprint (GP_ERR, "ERROR: ID vectors much be int\n"); + return (FALSE); + } + if (ID1vec->Nelements != ID2vec->Nelements) { + gprint (GP_ERR, "ERROR: ID vectors not all the same length\n"); + return (FALSE); + } + + // storage for the output and count vectors + int D_NOUT = 0.02*ID1vec->Nelements; + + int Nout = 0; + int NOUT = D_NOUT; + ResetVector (OUTvec, OPIHI_INT, NOUT); + if (CNTvec) { + ResetVector (CNTvec, OPIHI_INT, NOUT); + } + + // storage for the duplicate sequences + int NseqDup = 0; + int NSEQDUP = 10000; + if (SEQdup) { + ResetVector (SEQdup, OPIHI_INT, NSEQDUP); + } + + // incrementing pointers to the input IDs + opihi_int *ID1 = ID1vec->elements.Int; + opihi_int *ID2 = ID2vec->elements.Int; + + // generate the joined and sequence vectors + opihi_int *IDfull = NULL; + opihi_int *SEQdata = NULL; + ALLOCATE (IDfull, opihi_int, ID1vec->Nelements); + ALLOCATE (SEQdata, opihi_int, ID1vec->Nelements); + for (int i = 0; i < ID1vec->Nelements; i++) { + IDfull[i] = ID1[i] + (ID2[i] << 32); + SEQdata[i] = i; + } + + // XXX watch out: this all needs to get rationalized... + llsortpair ((off_t *)IDfull, (off_t *)SEQdata, ID1vec->Nelements); + + opihi_int *vtgt = OUTvec->elements.Int; + opihi_int *vsrc = IDfull; + + int onePercent = ID1vec->Nelements / 100; + + struct sigaction *old_sigaction = SetInterrupt(); + for (int i = 0; (i < ID1vec->Nelements) && !interrupt; Nout++) { + if (Nout >= NOUT) { + NOUT += D_NOUT; + REALLOCATE (vtgt, opihi_int, NOUT); + if (CNTvec) { + REALLOCATE (CNTvec->elements.Int, opihi_int, NOUT); + } + } + vtgt[Nout] = *vsrc; + int Ndup = 0; + opihi_int lastValue = *vsrc; + while ((i < ID1vec->Nelements) && (*vsrc == lastValue)) { + i++; + vsrc ++; + Ndup ++; + if (VERBOSE && (i % onePercent == 0)) gprint (GP_ERR, "."); + } + if (CNTvec) { + CNTvec->elements.Int[Nout] = Ndup; + } + if (SEQdup && (Ndup > 1)) { + // we want to save SEQdata values for the duplicates + // SEQdata[i-Ndup] .. SEQdata[i-N + for (int j = i - Ndup; j < i; j++) { + SEQdup->elements.Int[NseqDup] = SEQdata[j]; + NseqDup ++; + if (NseqDup >= NSEQDUP) { + NSEQDUP += 10000; + REALLOCATE (SEQdup->elements.Int, opihi_int, NSEQDUP); + } + } + } + } + ClearInterrupt (old_sigaction); + if (VERBOSE) gprint (GP_ERR, "\n"); + + free (SEQdata); + free (IDfull); + + if (SEQdup) { + SEQdup->Nelements = NseqDup; + REALLOCATE (SEQdup->elements.Int, opihi_int, NseqDup); + } + + // fix references and free up extra memory: + OUTvec->elements.Int = vtgt; + ResetVector (OUTvec, OPIHI_INT, Nout); + if (CNTvec) ResetVector (CNTvec, OPIHI_INT, Nout); + + return (TRUE); +} Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/vlorentz.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/vlorentz.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/vlorentz.c (revision 40477) @@ -0,0 +1,146 @@ +# include "data.h" + +/* local private functions */ +opihi_flt florentzOD (opihi_flt, opihi_flt *, int, opihi_flt *); + +# define GET_VAR(V,A) \ + c = get_variable (A); \ + if (c == NULL) { \ + gprint (GP_ERR, "missing fit parameter A\n"); \ + return (FALSE); \ + } \ + V = atof (c); \ + free (c); + +int vlorentz (int argc, char **argv) { + + int i, N, Npts, Npar, Quiet; + opihi_flt par[4], *v1, *v2, *dy, **covar; + opihi_flt chisq, ochisq, dchisq; + Vector *xvec, *yvec, *svec, *ovec; + char *c, name[16]; + + Quiet = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + Quiet = TRUE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-quiet"))) { + Quiet = TRUE; + remove_argument (N, &argc, argv); + } + +// Guess = FALSE; +// if ((N = get_argument (argc, argv, "-guess"))) { +// Guess = TRUE; +// remove_argument (N, &argc, argv); +// } + + if (argc != 5) { + gprint (GP_ERR, "USAGE: vlorentz (out)\n"); + gprint (GP_ERR, " may be the words 'con[stant]' or 'poi[sson]', in which case the error is constructed'\n"); + gprint (GP_ERR, " uses guesses: C0 (mean), C1 (sigma), C2 (norm), C3 (sky)\n"); + return (FALSE); + } + + if ((xvec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((yvec = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((ovec = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + CastVector (xvec, OPIHI_FLT); + CastVector (yvec, OPIHI_FLT); + + int Nsvec = strlen(argv[3]); + + if ((Nsvec >= 3) && (!strncasecmp ("poisson", argv[3], Nsvec) || !strncasecmp ("constant", argv[3], Nsvec))) { + svec = SelectVector ("vlorentz_err", ANYVECTOR, TRUE); + if (svec == NULL) return (FALSE); + MatchVector (svec, xvec, OPIHI_FLT); + v1 = svec[0].elements.Flt; + v2 = yvec[0].elements.Flt; + if (!strncasecmp ("poisson", argv[3], Nsvec)) { + for (i = 0; i < svec[0].Nelements; i++) { + v1[i] = (isfinite(v2[i]) && (v2[i] > 0)) ? sqrt(v2[i]) : 1.0; + } + } else { + for (i = 0; i < svec[0].Nelements; i++) { + v1[i] = 1.0; + } + } + } else { + if ((svec = SelectVector (argv[3], OLDVECTOR, TRUE)) == NULL) return (FALSE); + } + + CastVector (svec, OPIHI_FLT); + + Npts = xvec[0].Nelements; + ResetVector (ovec, OPIHI_FLT, Npts); + + ALLOCATE (dy, opihi_flt, Npts); + + GET_VAR (par[0], "C0"); + GET_VAR (par[1], "C1"); + GET_VAR (par[2], "C2"); + GET_VAR (par[3], "C3"); + Npar = 4; + + // mrqmin takes the inverse variance (do not generate NANs) + v1 = svec[0].elements.Flt; + v2 = dy; + for (i = 0; i < Npts; i++, v1++, v2++) { + *v2 = (*v1 == 0.0) ? 0.0 : 1.0 / (*v1 * *v1); + } + + ochisq = mrqinit (xvec[0].elements.Flt, yvec[0].elements.Flt, dy, Npts, par, Npar, florentzOD, !Quiet); + dchisq = ochisq + 2*Npts; + + for (i = 0; (i < 20) && ((dchisq > 0.1*(Npts - Npar)) || (dchisq <= 0.0)); i++) { + chisq = mrqmin (xvec[0].elements.Flt, yvec[0].elements.Flt, dy, Npts, par, Npar, florentzOD, !Quiet); + dchisq = ochisq - chisq; + ochisq = chisq; + if (!Quiet) gprint (GP_ERR, "dchisq: %f, Ndof: %d\n", dchisq, Npts - Npar); + } + if (!Quiet) gprint (GP_ERR, "%d iterations\n", i); + + for (i = 0; i < Npts; i++) { + ovec[0].elements.Flt[i] = florentzOD (xvec[0].elements.Flt[i], par, Npar, dy); + } + ovec[0].Nelements = Npts; + /* set output *before* variable renomalization */ + + covar = mrqcovar (Npar); + for (i = 0; i < Npar; i++) { + sprintf (name, "C%d", i); + set_variable (name, par[i]); + if (!Quiet) gprint (GP_ERR, "%d %f %f\n", i, par[i], sqrt(covar[i][i])); + sprintf (name, "dC%d", i); + set_variable (name, sqrt(covar[i][i])); + } + + free (dy); + mrqfree (Npar); + return (TRUE); +} + +/* pars: x_o, g, Io, sky */ +opihi_flt florentzOD (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar) { + OHANA_UNUSED_PARAM(Npar); + + opihi_flt z, f, L; + + opihi_flt xo = par[0]; + opihi_flt go = par[1]; + opihi_flt Io = par[2]; + opihi_flt So = par[3]; + + z = SQ(x - xo) + SQ(go)/4.0; + f = 1.0 / z; + L = 0.5*Io*go*f/M_PI + So; + + dpar[0] = Io*go*SQ(f)*(x - xo) / M_PI; + dpar[1] = 0.5*Io*(f - 0.5*SQ(f)*SQ(go)) / M_PI; + dpar[2] = 0.5*go*f / M_PI; + dpar[3] = 1; + + return (L); +} Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/write_vectors.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/write_vectors.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/write_vectors.c (revision 40477) @@ -165,7 +165,7 @@ } else { if (CSV) { - fprintf (f, "%d,", vec[j][0].elements.Int[i]); + fprintf (f, OPIHI_INT_FMT",", vec[j][0].elements.Int[i]); } else { - fprintf (f, "%d ", vec[j][0].elements.Int[i]); + fprintf (f, OPIHI_INT_FMT" ", vec[j][0].elements.Int[i]); } } Index: /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/zplot.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/zplot.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/cmd.data/zplot.c (revision 40477) @@ -117,5 +117,5 @@ /* point size determined by Zvec */ - graphmode.style = 2; /* plot points */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ graphmode.size = -1; /* point size determined by Zvec */ PlotVectorTriplet (kapa, xvec, yvec, &Zvec, mask, &graphmode); @@ -226,5 +226,5 @@ /* point size determined by Zvec */ - graphmode.style = 2; /* plot points */ + graphmode.style = KAPA_PLOT_POINTS; /* plot points */ graphmode.color = -1; /* point color determined by Zvec */ graphmode.etype = 0; /* no errorbars */ Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/Makefile =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/Makefile (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/Makefile (revision 40477) @@ -27,4 +27,5 @@ $(SRC)/mySequence.$(ARCH).o \ $(SRC)/photcode_ops.$(ARCH).o \ +$(SRC)/PeriodogramOps.$(ARCH).o \ $(SRC)/find_matches.$(ARCH).o @@ -45,4 +46,6 @@ $(SRC)/avextract.$(ARCH).o \ $(SRC)/avmatch.$(ARCH).o \ +$(SRC)/avperiodogram.$(ARCH).o \ +$(SRC)/avperiodomatch.$(ARCH).o \ $(SRC)/badimages.$(ARCH).o \ $(SRC)/catdir.$(ARCH).o \ Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/PeriodogramOps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/PeriodogramOps.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/PeriodogramOps.c (revision 40477) @@ -0,0 +1,220 @@ +# include "dvoshell.h" +# define MIN_VAR 1e-8 +# define NPERIODS 50000 + +static float minP = 0.2; +static float maxP = 200.0; + +void PeriodogramSetOptions (float minimumPeriod, float maximumPeriod) { + if (!isnan(minimumPeriod)) minP = minimumPeriod; + if (!isnan(maximumPeriod)) maxP = maximumPeriod; +} + +double percentileValue (double *values, int Nvalues, double percentile) { + + if (Nvalues <= 0) return NAN; + + if (percentile > 1.0) return NAN; + if (percentile < 0.0) return NAN; + + int Nm = Nvalues * percentile - 0.5; + int Np = Nm + 1; + + if (Nm < 0) return NAN; + if (Np < 0) return NAN; + if (Np >= Nvalues) return values[0]; + + double Pm = (Nm + 0.5) / Nvalues; + + double Pvalue = (percentile - Pm) * (values[Np] - values[Nm]) * Nvalues + values[Nm]; + return Pvalue; +} + +// low-level periodogram-fm function +PeriodogramResult *PeriodogramRawFloatingMean (opihi_flt *time, opihi_flt *flux, opihi_flt *dflux, int Npts) { + + /* find the max baseline, sum the inverse variances */ + double minT = time[0]; + double maxT = time[0]; + opihi_flt Weight = 0; + + // generate a sorted list of flux and calculate some percentile ranges + ALLOCATE_PTR (fluxSort, double, Npts); + for (int i = 0; i < Npts; i++) { + fluxSort[i] = flux[i]; + } + dsort (fluxSort, Npts); + + // I want two percentile ranges, e.g.,: 25 - 75, 5 - 95 + double P05 = percentileValue (fluxSort, Npts, 0.05); + double P25 = percentileValue (fluxSort, Npts, 0.25); + double P75 = percentileValue (fluxSort, Npts, 0.75); + double P95 = percentileValue (fluxSort, Npts, 0.95); + FREE (fluxSort); + + opihi_flt *tv = time; + opihi_flt *df = dflux; + opihi_flt *fv = flux; + + for (int i = 0; i < Npts; i++, tv++, df++) { + minT = MIN (minT, *tv); + maxT = MAX (maxT, *tv); + // add minimum variance to avoid dividing by 0.0 + Weight += 1.0 / (SQ(*df) + MIN_VAR); // MIN_VAR : added in quadrature to avoid Inf (XXX make this a user parameter?) + } + + double WeightInv = 1.0 / Weight; + double dTime = maxT - minT; + if (dTime == 0) { + gprint (GP_ERR, "ERROR: time range is zero\n"); + return NULL; + } + + int Np = 0; + int NP = NPERIODS; + ALLOCATE_PTR (period, double, NP); + ALLOCATE_PTR (power, double, NP); + + // for testing, we are going to write out the terms explicitly + // for optimization, some of the trig functions can be calculated more efficiently + // by storing cos,sin(w t) and using some recurrence rules + + // maxP = minP * dP^n [n = 0 -- Nperiods-1] + // log(maxP) = log(minP) + (Nperiods - 1) * log (dP) + // log(dP) = (log(maxP) - log(minP)) / (Nperiods - 1) + // double dP = LINEAR ? (maxP - minP)/(NPERIODS - 1) : pow(10.0, ((log10(maxP) - log10(minP))/(NPERIODS - 1))); + + double P = minP; + for (Np = 0; (Np < NPERIODS) && (P <= maxP); Np++) { + double w = 2*M_PI/P; + + /* find the period offset tau */ + tv = time; + df = dflux; + + double cs = 0.0, sn = 0.0, sn2 = 0.0, cs2 = 0.0; + + for (int i = 0; i < Npts; i++, tv++, df++) { + opihi_flt wt = WeightInv / (SQ(*df) + MIN_VAR); + cs += wt*cos (*tv*w); + sn += wt*sin (*tv*w); + cs2 += wt*cos (*tv*w*2); + sn2 += wt*sin (*tv*w*2); + } + double ytan = sn2 - 2*cs*sn; + double xtan = cs2 - (SQ(cs) - SQ(sn)); + double tau = 0.5*atan2 (ytan, xtan) / w; + + /* find the power at this period */ + tv = time; + df = dflux; + fv = flux; + + // YY = YY_s - Y_s*Y_s + // CC = CC_s - C_s*C_s + // SS = SS_s - S_s*S_s + // YC = YC_s - Y_s*C_s + // YS = YS_s - Y_s*S_s + + double YY_s = 0.0, CC_s = 0.0, SS_s = 0.0, YC_s = 0.0, YS_s = 0.0; + double Y_s = 0.0, C_s = 0.0, S_s = 0.0; + for (int i = 0; i < Npts; i++, tv++, fv++, df++) { + opihi_flt wt = WeightInv / (SQ(*df) + MIN_VAR); + cs = cos (w*(*tv-tau)); + sn = sin (w*(*tv-tau)); + + double wtf = *fv*wt; + Y_s += wtf; + C_s += wt*cs; + S_s += wt*sn; + + YY_s += wtf*(*fv); + YC_s += wtf*cs; + YS_s += wtf*sn; + + CC_s += wt*cs*cs; + SS_s += wt*sn*sn; + } + double YY = YY_s - SQ(Y_s); + double YC = YC_s - Y_s*C_s; + double YS = YS_s - Y_s*S_s; + double CC = CC_s - SQ(C_s); + double SS = SS_s - SQ(S_s); + + double Pa = SQ(YC) / CC; + double Pb = SQ(YS) / SS; + double Po = (Pa + Pb) / YY; + + power[Np] = Po; + period[Np] = P; + if (Np >= NP) { + NP += 100; + REALLOCATE (power, opihi_flt, NP); + REALLOCATE (period, opihi_flt, NP); + } + + float dP = 0.4 * SQ(P) / dTime; + P += dP; + } + + ALLOCATE_PTR (result, PeriodogramResult, 1); + result->Nmeasure = Npts; + result->Nperiods = Np; + result->period = period; + result->power = power; + + result->P50 = 0.5*(P75 + P25); + result->R50 = P75 - P25; + result->R90 = P95 - P05; + + return result; +} + +void PeriodogramResultFree (PeriodogramResult *result) { + if (!result) return; + + FREE (result->period); + FREE (result->power); + free (result); + return; +} + +// write the period and power vectors to an extension in this output file +void PeriodogramResultSave (PeriodogramResult *result, char *extname, FILE *foutput, Average *average) { + + // XXX add some metadata to the header: + // Nmeasurements (add into the results structure), g, r, i, z, y? + + // generate a binary table + Header outheader; + FTable outtable; + gfits_init_header (&outheader); + gfits_init_table (&outtable); + outtable.header = &outheader; + + gfits_create_table_header (&outheader, "BINTABLE", extname); + + // add some metadata derived from average: + gfits_modify (&outheader, "RA_OBJ", "%lf", 1, average->R); + gfits_modify (&outheader, "DEC_OBJ", "%lf", 1, average->D); + gfits_modify (&outheader, "OBJ_ID", "%d", 1, average->objID); + gfits_modify (&outheader, "CAT_ID", "%d", 1, average->catID); + gfits_modify (&outheader, "NMEASURE", "%d", 1, result->Nmeasure); + gfits_modify (&outheader, "P50", "%lf", 1, result->P50); + gfits_modify (&outheader, "R50", "%lf", 1, result->R50); + gfits_modify (&outheader, "R90", "%lf", 1, result->R90); + gfits_define_bintable_column (&outheader, "D", "PERIOD", NULL, NULL, 1.0, 0.0); + gfits_define_bintable_column (&outheader, "D", "POWER", NULL, NULL, 1.0, 0.0); + + gfits_create_table (&outheader, &outtable); + + gfits_set_bintable_column (&outheader, &outtable, "PERIOD", result->period, result->Nperiods); + gfits_set_bintable_column (&outheader, &outtable, "POWER", result->power, result->Nperiods); + + // write the actual table data + gfits_fwrite_Theader (foutput, &outheader); + gfits_fwrite_table (foutput, &outtable); + + gfits_free_header (&outheader); + gfits_free_table (&outtable); +} Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/avperiodogram.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/avperiodogram.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/avperiodogram.c (revision 40477) @@ -0,0 +1,340 @@ +# include "dvoshell.h" + +int avperiodogram (int argc, char **argv) { + + int N, next, Nfields; + + dbStack *stack = NULL; + dbField *fields = NULL; + dbValue *values = NULL; + SkyList *skylist = NULL; + SkyRegionSelection *selection = NULL; + + // **** parse the optional arguments **** + if ((N = get_argument (argc, argv, "-h"))) goto help; + if ((N = get_argument (argc, argv, "--help"))) goto help; + + int VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + remove_argument (N, &argc, argv); + VERBOSE = TRUE; + } + int VERBOSE2 = FALSE; + if ((N = get_argument (argc, argv, "-vv"))) { + remove_argument (N, &argc, argv); + VERBOSE = TRUE; + VERBOSE2 = TRUE; + } + + int PARALLEL = FALSE; + if ((N = get_argument (argc, argv, "-parallel"))) { + remove_argument (N, &argc, argv); + PARALLEL = TRUE; + } + + /* load photcode information (this needs to come before the SelectRegions command below to define the catdir */ + if (!InitPhotcodes ()) goto escape; + int Nsecfilt = GetPhotcodeNsecfilt (); + + // parse skyregion options. NOTE: this is stripped off in parallel operation and always + // defined for the client via the -skyregion option. The dvo_client parses this + // argument in the main program, before it is passed to the command (like mextract) + if ((selection = SetRegionSelection (&argc, argv)) == NULL) { + print_error(); + goto escape; + } + + /* load region corresponding to selection above */ + if ((skylist = SelectRegions (selection)) == NULL) goto escape; + + // **** launch parallel / remote jobs **** + // the result files are left behind for the user to access (no automatic re-merge) + if (PARALLEL && !HOST_ID) { + int status = HostTableParallelOps (skylist, argc, argv, NULL, FALSE, 0, VERBOSE); + SkyListFree (skylist); + FreeSkyRegionSelection (selection); + return status; + } + + // NOTE: optional arguments below are parsed after the possible remote call so they are also passed to the dvo_client + + if ((N = get_argument (argc, argv, "-min-period"))) { + remove_argument (N, &argc, argv); + PeriodogramSetOptions (atof (argv[N]), NAN); + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-max-period"))) { + remove_argument (N, &argc, argv); + PeriodogramSetOptions (NAN, atof (argv[N])); + remove_argument (N, &argc, argv); + } + + // the first three Measure fields must be time, mag, dmag (mag:err) + int Nmfields = 3; + ALLOCATE_PTR (mfields, dbField, Nmfields); + dbInitField (&mfields[0]); ParseMeasureField (&mfields[0], "time"); + dbInitField (&mfields[1]); ParseMeasureField (&mfields[1], "mag"); + dbInitField (&mfields[2]); ParseMeasureField (&mfields[2], "mag:err"); + + int Nmstack = 0; + dbStack *mstack = NULL; + dbValue *mvalues = NULL; + + // we can restrict these with a -where-measure clause + if ((N = get_argument (argc, argv, "-where-measure"))) { + remove_argument (N, &argc, argv); + + // parse the remainder of the line as a boolean math expression + unsigned int Nmcstack; + char **mcstack = isolate_elements (argc-N, &argv[N], &Nmcstack); + + // construct the db Boolean math stack (frees cmstack) + mstack = dbRPN (Nmcstack, mcstack, &Nmstack); + if (Nmcstack && !Nmstack) { + print_error(); + goto escape; + } + + // parse stack elements into fields and scalars as needed (supplement mfields) + if (!dbCheckStack (mstack, Nmstack, DVO_TABLE_MEASURE, &mfields, &Nmfields)) goto escape; + + argc = N; // hide remaining entries from the rest of the code below + } + // output values + ALLOCATE (mvalues, dbValue, Nmfields); + + // use remote tables, but not dvo_client.. + int PARALLEL_LOCAL = FALSE; + HostTable *table = NULL; + if ((N = get_argument (argc, argv, "-parallel-local"))) { + remove_argument (N, &argc, argv); + PARALLEL_LOCAL = TRUE; + + char *CATDIR = GetCATDIR(); + if (!CATDIR) { + gprint (GP_ERR, "CATDIR is not set\n"); + return FALSE; + } + SkyTable *sky = GetSkyTable(); + if (!sky) { + gprint (GP_ERR, "failed to load sky table for database\n"); + return FALSE; + } + table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + gprint (GP_ERR, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + return FALSE; + } + } + + // command-line is of the form: avperiodogram (output) where (field op value)... + if (argc < 2) goto help; + char *output = strcreate (argv[1]); + + // **** generate output file **** + // we save the results in a single FITS file, one extension per object + FILE *foutput = NULL; + if (RESULT_FILE) { + foutput = fopen (RESULT_FILE, "w"); + } else { + foutput = fopen (output, "w"); + } + // generate the PHU and write to disk + // XXX add some metadata here? + Header header; + Matrix matrix; + gfits_init_header (&header); + gfits_init_matrix (&matrix); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + gfits_fwrite_header (foutput, &header); + gfits_fwrite_matrix (foutput, &matrix); + gfits_free_header (&header); + gfits_free_matrix (&matrix); + + Catalog catalog; + dvo_catalog_init (&catalog, TRUE); + + // init locally static variables (time refs) + dbExtractAveragesInit (); + dbExtractMeasuresInit(HOST_ID); + + // examine line for 'where' or 'match to'. 'match to' is forbidden + int state = dbCmdlineConditions (argc, argv, 2, &next); + if (state == DVO_DB_CMDLINE_ERROR) goto escape; + if (state == DVO_DB_CMDLINE_IS_MATCH) goto escape; // not allowed for avperiodogram + + // parse the remainder of the line as a boolean math expression + unsigned int Ncstack; + char **cstack = isolate_elements (argc-next, &argv[next], &Ncstack); + + // construct the db Boolean math stack (frees cstack) + int Nstack; + stack = dbRPN (Ncstack, cstack, &Nstack); + if (Ncstack && !Nstack) { + print_error(); + goto escape; + } + + // add the skyregion limits to the where statement (or create) + dbAstroRegionLimits (&stack, &Nstack, selection, DVO_TABLE_AVERAGE); + + // parse stack elements into fields and scalars as needed + Nfields = 0; + ALLOCATE (fields, dbField, 1); + if (!dbCheckStack (stack, Nstack, DVO_TABLE_AVERAGE, &fields, &Nfields)) goto escape; + + // output values + ALLOCATE (values, dbValue, Nfields); + + int Nobject = 0; + char extname[80]; + + // grab data from all selected sky regions + struct sigaction *old_sigaction = SetInterrupt(); + for (off_t i = 0; (i < skylist[0].Nregions) && !interrupt; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue; + + /* lock, load, unlock catalog */ + char hostfile[1024]; + if (PARALLEL_LOCAL) { + int hostID = (skylist[0].regions[i]->hostFlags & DATA_USE_BCK) ? skylist[0].regions[i]->backupID : skylist[0].regions[i]->hostID; + int seq = table->index[hostID]; + HOSTDIR = table->hosts[seq].pathname; + } + snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name); + + dvo_catalog_init (&catalog, TRUE); + catalog.filename = (HOST_ID) ? hostfile : skylist[0].filename[i]; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE; + catalog.Nsecfilt = 0; + + if (VERBOSE) gprint (GP_ERR, "trying %s ("OFF_T_FMT" of "OFF_T_FMT")\n", catalog.filename, i, skylist[0].Nregions); + + // an error exit status here is a significant error + if (!dvo_catalog_open (&catalog, NULL, VERBOSE2, "r")) { + gprint (GP_ERR, "ERROR: failure to open catalog file %s\n", catalog.filename); + exit (2); + } + dvo_catalog_unlock (&catalog); + + // Scan the catalog for objects which match the WHERE clause + for (off_t j = 0; (j < catalog.Naverage) && !interrupt; j++) { + dbExtractAveragesInitAve (); // reset counters for saved fields (costs very little) + + Average *average = &catalog.average[j]; + + off_t mOff = average->measureOffset; + Measure *measure = &catalog.measure[mOff]; + + off_t nSec = j*Nsecfilt; + SecFilt *secfilt = &catalog.secfilt[nSec]; + + // extract the values needed to evaluate the WHERE clause + for (off_t n = 0; n < Nfields; n++) { + // for Measure, we are passing in the *first* measure, but average->Nmeasure gives the count + // for Secfilt, we are passing in the first entry; photcode.equiv gives the entry + // for Lensobj, we are passing in the first entry or NULL; photcode.equiv gives the entry + values[n] = dbExtractAverages (average, secfilt, measure, NULL, NULL, NULL, &fields[n]); + } + + // test the conditional statement + if (!dbBooleanCond (stack, Nstack, values)) continue; + + // generate time (mjd), mag, dmag vectors + ALLOCATE_PTR (time, opihi_flt, average->Nmeasure); + ALLOCATE_PTR (mag, opihi_flt, average->Nmeasure); + ALLOCATE_PTR (dmag, opihi_flt, average->Nmeasure); + + int Npts = 0; + + // this object passes the conditions above: now grab its measures which match our conditions? + for (off_t k = 0; (k < average->Nmeasure); k++) { + if (measure[k].averef != j) { + gprint (GP_ERR, "ERROR: inconsistent measure->average link. Unsorted database?\n"); + goto escape; + } + + // extract the relevant values for this measurement + dbExtractMeasuresInitMeas (); // reset counters for saved fields (costs very little + + // XXX I need to have a different boolean expression here to restrict the measurements + // The first 3 values[] / fields[] will be time, mag, dmag + for (off_t n = 0; n < Nmfields; n++) { + mvalues[n] = dbExtractMeasures (average, secfilt, &measure[k], NULL, NULL, &mfields[n]); + } + if (!dbBooleanCond (mstack, Nmstack, mvalues)) continue; + + // do not allow any NAN or Inf values to polute the result + if (!isfinite(mvalues[0].Flt)) continue; + if (!isfinite(mvalues[1].Flt)) continue; + if (!isfinite(mvalues[2].Flt)) continue; + + time[Npts] = mvalues[0].Flt; + mag[Npts] = mvalues[1].Flt; + dmag[Npts] = mvalues[2].Flt; + + Npts++; + } + + PeriodogramResult *result = PeriodogramRawFloatingMean (time, mag, dmag, Npts); + free (time); + free (mag); + free (dmag); + + if (!result) continue; + + if (VERBOSE2) gprint (GP_ERR, "periodogram for %d, %d\n", (int) average->catID, (int) average->objID); + + // XXX if we have 1e6 objects, we will have output files with sizes of ~80GB + // XXX warn or exit if Nboject > 1e6? + snprintf (extname, 80, "OBJ_%06d", Nobject); + PeriodogramResultSave (result, extname, foutput, average); + PeriodogramResultFree (result); + Nobject ++; + } + dvo_catalog_free (&catalog); + } + ClearInterrupt (old_sigaction); + + fclose (foutput); + fflush (foutput); + + // free measure stack stuff + FREE (mvalues); + dbFreeFields(mfields, Nmfields); + dbFreeStack(mstack, Nmstack); + FREE (mstack); + + FREE (values); + dbFreeFields (fields, Nfields); + dbFreeStack (stack, Nstack); + FREE (stack); + + SkyListFree (skylist); + FreeSkyRegionSelection (selection); + + free (output); + return (TRUE); + + escape: + if (values) free (values); + dbFreeFields (fields, Nfields); + dbFreeStack (stack, Nstack); + if (stack) free (stack); + SkyListFree (skylist); + FreeSkyRegionSelection (selection); + dvo_catalog_free (&catalog); + return (FALSE); + + + help: + gprint (GP_ERR, "USAGE: avperiodogram (output) where (expression) -where-measure (expression)\n"); + gprint (GP_ERR, " the where (expression) is a boolean to limit the objects analysed based on average properties\n"); + gprint (GP_ERR, " NOTE: the optional -where-measure (expression) MUST come after the average where expression\n"); + return (FALSE); +} Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/avperiodomatch.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/avperiodomatch.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/avperiodomatch.c (revision 40477) @@ -0,0 +1,370 @@ +# include "dvoshell.h" + +int avperiodomatch (int argc, char **argv) { + + int N; + + SkyList *skylist = NULL; + SkyRegionSelection *selection = NULL; + + int Ninvec = 0; + + Vector **invec = NULL; + Vector *RAvec = NULL; + Vector *DECvec = NULL; + + // **** parse the optional arguments **** + if ((N = get_argument (argc, argv, "-h"))) goto help; + if ((N = get_argument (argc, argv, "--help"))) goto help; + + int VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + remove_argument (N, &argc, argv); + VERBOSE = TRUE; + } + int VERBOSE2 = FALSE; + if ((N = get_argument (argc, argv, "-vv"))) { + remove_argument (N, &argc, argv); + VERBOSE = TRUE; + VERBOSE2 = TRUE; + } + + int PARALLEL = FALSE; + if ((N = get_argument (argc, argv, "-parallel"))) { + remove_argument (N, &argc, argv); + PARALLEL = TRUE; + } + + /* load photcode information (this needs to come before the SelectRegions command below to define the catdir */ + if (!InitPhotcodes ()) goto escape; + int Nsecfilt = GetPhotcodeNsecfilt (); + + // parse skyregion options. NOTE: this is stripped off in parallel operation and always + // defined for the client via the -skyregion option. The dvo_client parses this + // argument in the main program, before it is passed to the command (like mextract) + if ((selection = SetRegionSelection (&argc, argv)) == NULL) { + print_error(); + goto escape; + } + + /* load region corresponding to selection above */ + if ((skylist = SelectRegions (selection)) == NULL) goto escape; + + // dump results directly to fits file (esp for parallel dvo) + char *CoordsFile = NULL; + if ((N = get_argument (argc, argv, "-coords"))) { + remove_argument (N, &argc, argv); + CoordsFile = strcreate(argv[N]); + remove_argument (N, &argc, argv); + } + + // XXX arg-parsing is a bit confusing with the -where-measure option + if (!CoordsFile && (argc < 5)) goto help; + if ( CoordsFile && (argc < 3)) goto help; + + RAvec = NULL; + DECvec = NULL; + // get vectors corresponding to coordinates of interest + if (CoordsFile) { + // read RAvec, DECvec from coords file (1st 2 fields?) + Ninvec = 0; + invec = ReadVectorTableFITS (CoordsFile, "COORDS", &Ninvec); + RAvec = invec[0]; + DECvec = invec[1]; + } else { + if ((RAvec = SelectVector (argv[2], ANYVECTOR, TRUE)) == NULL) goto help; + if ((DECvec = SelectVector (argv[3], ANYVECTOR, TRUE)) == NULL) goto help; + remove_argument (2, &argc, argv); // strip off RA + remove_argument (2, &argc, argv); // strip off DEC + } + + // **** launch parallel / remote jobs **** + // the result files are left behind for the user to access (no automatic re-merge) + if (PARALLEL && !HOST_ID) { + // allocate the temp array and copy all but (RA) (DEC) + int targc = 0; + char **targv = NULL; + ALLOCATE (targv, char *, argc + 2); + for (int i = 0; i < argc; i++) { + targv[targc] = strcreate (argv[i]); + targc ++; + } + + // if not specified, create the coords.fits input file + // NOTE: RAvec, DECvec were set above + if (!CoordsFile) { + ALLOCATE_PTR (vec, Vector *, 2); + vec[0] = RAvec; + vec[1] = DECvec; + + CoordsFile = abspath("coords.fits", 1024); + int status = WriteVectorTableFITS (CoordsFile, "COORDS", vec, 2, FALSE, FALSE, NULL, 0); + if (!status) goto escape; + free (vec); + } + + // add the coords file to the args list + targv[targc+0] = strcreate ("-coords"); + targv[targc+1] = CoordsFile; // this gets freed with targv + targc += 2; + + int status = HostTableParallelOps (skylist, targc, targv, NULL, FALSE, RAvec->Nelements, VERBOSE); + + // free up targv + for (int i = 0; i < targc; i++) { + free (targv[i]); + } + free (targv); + + SkyListFree (skylist); + FreeSkyRegionSelection (selection); + return status; + } + + // NOTE: optional arguments below are parsed after the possible remote call so they are also passed to the dvo_client above + + if ((N = get_argument (argc, argv, "-min-period"))) { + remove_argument (N, &argc, argv); + PeriodogramSetOptions (atof (argv[N]), NAN); + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-max-period"))) { + remove_argument (N, &argc, argv); + PeriodogramSetOptions (NAN, atof (argv[N])); + remove_argument (N, &argc, argv); + } + + // the first three Measure fields must be time, mag, dmag (mag:err) + int Nmfields = 3; + ALLOCATE_PTR (mfields, dbField, Nmfields); + dbInitField (&mfields[0]); ParseMeasureField (&mfields[0], "time"); + dbInitField (&mfields[1]); ParseMeasureField (&mfields[1], "mag"); + dbInitField (&mfields[2]); ParseMeasureField (&mfields[2], "mag:err"); + + int Nmstack = 0; + dbStack *mstack = NULL; + dbValue *mvalues = NULL; + + // we can restrict these with a -where-measure clause + if ((N = get_argument (argc, argv, "-where-measure"))) { + remove_argument (N, &argc, argv); + + // parse the remainder of the line as a boolean math expression + unsigned int Nmcstack; + char **mcstack = isolate_elements (argc-N, &argv[N], &Nmcstack); + + // construct the db Boolean math stack (frees cmstack) + mstack = dbRPN (Nmcstack, mcstack, &Nmstack); + if (Nmcstack && !Nmstack) { + print_error(); + goto escape; + } + + // parse stack elements into fields and scalars as needed (supplement mfields) + if (!dbCheckStack (mstack, Nmstack, DVO_TABLE_MEASURE, &mfields, &Nmfields)) goto escape; + + argc = N; // hide remaining entries from the rest of the code below + } + // output values + ALLOCATE (mvalues, dbValue, Nmfields); + + // use remote tables, but not dvo_client.. + int PARALLEL_LOCAL = FALSE; + HostTable *table = NULL; + if ((N = get_argument (argc, argv, "-parallel-local"))) { + remove_argument (N, &argc, argv); + PARALLEL_LOCAL = TRUE; + + char *CATDIR = GetCATDIR(); + if (!CATDIR) { + gprint (GP_ERR, "CATDIR is not set\n"); + return FALSE; + } + SkyTable *sky = GetSkyTable(); + if (!sky) { + gprint (GP_ERR, "failed to load sky table for database\n"); + return FALSE; + } + table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + gprint (GP_ERR, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + return FALSE; + } + } + + // command-line is of the form: avperiodogram (output) where (field op value)... + if (argc != 3) goto help; + + char *output = strcreate (argv[1]); + float RADIUS = atof (argv[2]); + + // **** generate output file **** + // we save the results in a single FITS file, one extension per object + FILE *foutput = NULL; + if (RESULT_FILE) { + foutput = fopen (RESULT_FILE, "w"); + } else { + foutput = fopen (output, "w"); + } + // generate the PHU and write to disk + // XXX add some metadata here? + Header header; + Matrix matrix; + gfits_init_header (&header); + gfits_init_matrix (&matrix); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + gfits_fwrite_header (foutput, &header); + gfits_fwrite_matrix (foutput, &matrix); + gfits_free_header (&header); + gfits_free_matrix (&matrix); + + Catalog catalog; + dvo_catalog_init (&catalog, TRUE); + + // init locally static variables (time refs) + dbExtractMeasuresInit(HOST_ID); + + int Nobject = 0; + char extname[80]; + + int NPTS = RAvec->Nelements; + ALLOCATE_PTR (idxValue, off_t, NPTS); + + // grab data from all selected sky regions + struct sigaction *old_sigaction = SetInterrupt(); + for (off_t i = 0; (i < skylist[0].Nregions) && !interrupt; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue; + + /* lock, load, unlock catalog */ + char hostfile[1024]; + if (PARALLEL_LOCAL) { + int hostID = (skylist[0].regions[i]->hostFlags & DATA_USE_BCK) ? skylist[0].regions[i]->backupID : skylist[0].regions[i]->hostID; + int seq = table->index[hostID]; + HOSTDIR = table->hosts[seq].pathname; + } + snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name); + + dvo_catalog_init (&catalog, TRUE); + catalog.filename = (HOST_ID) ? hostfile : skylist[0].filename[i]; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE; + catalog.Nsecfilt = 0; + + if (VERBOSE) gprint (GP_ERR, "trying %s ("OFF_T_FMT" of "OFF_T_FMT")\n", catalog.filename, i, skylist[0].Nregions); + + // an error exit status here is a significant error + if (!dvo_catalog_open (&catalog, NULL, VERBOSE2, "r")) { + gprint (GP_ERR, "ERROR: failure to open catalog file %s\n", catalog.filename); + exit (2); + } + dvo_catalog_unlock (&catalog); + + find_matches_by_vectors (skylist[0].regions[i], &catalog, RAvec, DECvec, RADIUS, idxValue); + + // Scan the catalog for objects which match the WHERE clause + for (off_t j = 0; (j < NPTS) && !interrupt; j++) { + off_t Ncat = idxValue[j]; + + if (Ncat == -1) continue; // this point is not in this catalog file + if (Ncat == -2) continue; // no matches to this point + + dbExtractAveragesInitAve (); // reset counters for saved fields (costs very little) + + Average *average = &catalog.average[Ncat]; + + Measure *measure = &catalog.measure[average->measureOffset]; + + off_t m = Ncat*Nsecfilt; + SecFilt *secfilt = &catalog.secfilt[m]; + + // generate time (mjd), mag, dmag vectors + ALLOCATE_PTR (time, opihi_flt, average->Nmeasure); + ALLOCATE_PTR (mag, opihi_flt, average->Nmeasure); + ALLOCATE_PTR (dmag, opihi_flt, average->Nmeasure); + + int Npts = 0; + + // this object passes the conditions above: now grab its measures which match our conditions? + for (off_t k = 0; (k < average->Nmeasure); k++) { + if (measure[k].averef != Ncat) { + gprint (GP_ERR, "ERROR: inconsistent measure->average link. Unsorted database?\n"); + goto escape; + } + + // extract the relevant values for this measurement + dbExtractMeasuresInitMeas (); // reset counters for saved fields (costs very little + + // XXX I need to have a different boolean expression here to restrict the measurements + // The first 3 values[] / fields[] will be time, mag, dmag + for (off_t n = 0; n < Nmfields; n++) { + mvalues[n] = dbExtractMeasures (average, secfilt, &measure[k], NULL, NULL, &mfields[n]); + } + if (!dbBooleanCond (mstack, Nmstack, mvalues)) continue; + + // do not allow any NAN or Inf values to polute the result + if (!isfinite(mvalues[0].Flt)) continue; + if (!isfinite(mvalues[1].Flt)) continue; + if (!isfinite(mvalues[2].Flt)) continue; + + time[Npts] = mvalues[0].Flt; + mag[Npts] = mvalues[1].Flt; + dmag[Npts] = mvalues[2].Flt; + + Npts++; + } + + PeriodogramResult *result = PeriodogramRawFloatingMean (time, mag, dmag, Npts); + free (time); + free (mag); + free (dmag); + + if (!result) continue; + + if (VERBOSE2) gprint (GP_ERR, "periodogram for %d, %d\n", (int) average->catID, (int) average->objID); + + // XXX if we have 1e6 objects, we will have output files with sizes of ~80GB + // XXX warn or exit if Nboject > 1e6? + snprintf (extname, 80, "OBJ_%06d", Nobject); + PeriodogramResultSave (result, extname, foutput, average); + PeriodogramResultFree (result); + Nobject ++; + } + dvo_catalog_free (&catalog); + } + ClearInterrupt (old_sigaction); + + fclose (foutput); + fflush (foutput); + + FREE (idxValue); + + // free measure stack stuff + FREE (mvalues); + dbFreeFields(mfields, Nmfields); + dbFreeStack(mstack, Nmstack); + FREE (mstack); + + SkyListFree (skylist); + FreeSkyRegionSelection (selection); + + free (output); + return (TRUE); + + escape: + SkyListFree (skylist); + FreeSkyRegionSelection (selection); + dvo_catalog_free (&catalog); + + if (invec) FreeVectorArray (invec, Ninvec); + return (FALSE); + + + help: + gprint (GP_ERR, "USAGE: avperiodomatch (output) (RA) (DEC) (RADIUS) -where-measure (expression)\n"); + gprint (GP_ERR, " NOTE: the optional -where-measure (expression) MUST come after the required arguments\n"); + return (FALSE); +} Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/catalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/catalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/catalog.c (revision 40477) @@ -390,5 +390,5 @@ REALLOCATE (Zvec.elements, float, MAX (Zvec.Nelements, 1)); - graphmode.style = 2; /* set style to points */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ graphmode.size = -1; /* point size determined by Zvec */ graphmode.etype = 0; /* no errorbars */ Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/dmt.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/dmt.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/dmt.c (revision 40477) @@ -150,5 +150,5 @@ vec5[0].Nelements = Dvec.Nelements; } else { - graphmode.style = 2; /* set style to points */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ PlotVector (kapa, N, Xvec.elements, Yvec.elements, &graphmode); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/dvo_host_utils.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/dvo_host_utils.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/dvo_host_utils.c (revision 40477) @@ -212,6 +212,6 @@ continue; } - free (table->hosts[i].results); - table->hosts[i].results = NULL; + // free (table->hosts[i].results); + // table->hosts[i].results = NULL; set_int_variable (name, 1); // result file has been read @@ -225,4 +225,7 @@ FreeVectorArray (invec, Ninvec); } + } else { + // free (table->hosts[i].results); + // table->hosts[i].results = NULL; } } @@ -242,5 +245,5 @@ free (vec); - free (table); + FreeHostTable (table); return TRUE; } @@ -284,5 +287,5 @@ // XXX a bit of a waste (but only 1024 * 60 bytes or so ALLOCATE (table->hosts[i].results, char, DVO_MAX_PATH); - snprintf (table->hosts[i].results, DVO_MAX_PATH, "%s/dvo.results.%s.fits", table->hosts[i].pathname, uniquer); + snprintf (table->hosts[i].results, DVO_MAX_PATH, "%s/dvo.results.%s.%04d.fits", table->hosts[i].pathname, uniquer, table->hosts[i].hostID); int Ninvec = 0; Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/fitcolors.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/fitcolors.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/fitcolors.c (revision 40477) @@ -48,4 +48,6 @@ if (!SetPhotSelections (&argc, argv, 4)) goto usage; + int textcolor = KapaColorByName ("black"); + // range for valid data points (exclude extreme outliers) minDelta = -0.2; @@ -92,6 +94,6 @@ graphdata.ymin = minDelta; graphdata.ymax = maxDelta; - graphdata.style = 2; - graphdata.ptype = 2; + graphmode.style = KAPA_PLOT_POINTS; /* points */ + graphmode.ptype = KAPA_POINT_CIRCLE_SOLID; /* connect pairs of points */ KapaClearSections (kapa); KapaSetFont (kapa, "helvetica", 14); @@ -192,6 +194,6 @@ graphdata.ymin = minDelta; graphdata.ymax = maxDelta; - graphdata.style = 2; - graphdata.ptype = 2; + graphmode.style = KAPA_PLOT_POINTS; /* points */ + graphmode.ptype = KAPA_POINT_CIRCLE_SOLID; /* connect pairs of points */ } @@ -315,5 +317,5 @@ deltaFit[i] = C0 + C1*colorFit[i]; } - graphdata.style = 0; + graphmode.style = KAPA_PLOT_CONNECT; /* lines */ graphdata.color = KapaColorByName ("red"); @@ -324,10 +326,10 @@ KapaSetFont (kapa, "helvetica", 8); sprintf (label, "%s", code[0][0].name); - KapaSendTextline (kapa, label, 0.2*maxColor + 0.8*minColor, 0.8*maxDelta + 0.2*minDelta, 0.0); + KapaSendTextline (kapa, label, 0.2*maxColor + 0.8*minColor, 0.8*maxDelta + 0.2*minDelta, 0.0, textcolor); sprintf (label, "%s", code[1][0].name); - KapaSendTextline (kapa, label, 0.2*maxColor + 0.8*minColor, 0.2*maxDelta + 0.8*minDelta, 0.0); + KapaSendTextline (kapa, label, 0.2*maxColor + 0.8*minColor, 0.2*maxDelta + 0.8*minDelta, 0.0, textcolor); KapaSetFont (kapa, "helvetica", 14); - graphdata.style = 2; + graphmode.style = KAPA_PLOT_POINTS; /* points */ graphdata.color = KapaColorByName ("black"); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/fitsed.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/fitsed.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/fitsed.c (revision 40477) @@ -157,6 +157,6 @@ if (!GetGraph (&graphdata, &kapa, NULL)) return (FALSE); SetLimitsRaw (wavecode, NULL, Nfilter, &graphdata); - graphdata.style = 2; - graphdata.ptype = 2; + graphdata.style = KAPA_PLOT_POINTS; /* points */ + graphdata.ptype = KAPA_POINT_CIRCLE_SOLID; KapaClearSections (kapa); magSection.name = strcreate ("mag"); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/gimages.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/gimages.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/gimages.c (revision 40477) @@ -214,9 +214,9 @@ if (PixelCoords) { gprint (GP_LOG, "%3d %5d %s %6.1f %6.1f %20s %5d %2d %4.2f %6.3f %5.3f %5.3f %4x %7d\n", - Nfound, (int) i, image[i].name, X, Y, date, image[i].nstar, image[i].photcode, image[i].secz, image[i].Mcal, image[i].dMcal, image[i].exptime, image[i].flags, image[i].imageID); + Nfound, (int) i, image[i].name, X, Y, date, image[i].nstar, image[i].photcode, image[i].secz, image[i].McalPSF, image[i].dMcal, image[i].exptime, image[i].flags, image[i].imageID); } else { XY_to_RD (&ra, &dec, 0.5*image[i].NX, 0.5*image[i].NY, &image[i].coords); gprint (GP_LOG, "%3d %5d %s %8.4f %8.4f %20s %5d %2d %4.2f %6.3f %5.3f %5.3f %4x %7d\n", - Nfound, (int) i, image[i].name, ra, dec, date, image[i].nstar, image[i].photcode, image[i].secz, image[i].Mcal, image[i].dMcal, image[i].exptime, image[i].flags, image[i].imageID); + Nfound, (int) i, image[i].name, ra, dec, date, image[i].nstar, image[i].photcode, image[i].secz, image[i].McalPSF, image[i].dMcal, image[i].exptime, image[i].flags, image[i].imageID); } sprintf (name, "IMAGEx:%d", Nfound); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/gstar.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/gstar.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/gstar.c (revision 40477) @@ -760,8 +760,11 @@ if (FULL_OUTPUT) { - gprint (GP_LOG, "%6.3f ", catalog.measure[Nv].Mcal); + gprint (GP_LOG, "%6.3f ", catalog.measure[Nv].McalPSF); + gprint (GP_LOG, "%6.3f ", catalog.measure[Nv].McalAPER); gprint (GP_LOG, "%6.3f ", catalog.measure[Nv].Mflat); - gprint (GP_LOG, "%6.3f ", catalog.measure[Nv].Map); - gprint (GP_LOG, "%6.3f ", catalog.measure[Nv].Mkron); + Mrel = PhotRel (&catalog.measure[Nv], &catalog.average[k], &catalog.secfilt[k*Nsecfilt], MAG_CLASS_APER); + gprint (GP_LOG, "%6.3f ", Mrel); + Mrel = PhotRel (&catalog.measure[Nv], &catalog.average[k], &catalog.secfilt[k*Nsecfilt], MAG_CLASS_KRON); + gprint (GP_LOG, "%6.3f ", Mrel); gprint (GP_LOG, "%6.3f ", catalog.measure[Nv].dMkron); gprint (GP_LOG, "%5.1f ", pow(10.0, 0.4*catalog.measure[Nv].dt)); @@ -960,5 +963,5 @@ print_double (NAN); } else { - print_double_exp (secfilt[seq].Mstdev); + print_double_exp (secfilt[seq].sMpsfChp); } break; @@ -1023,5 +1026,5 @@ print_double (NAN); } else { - print_double (secfilt[seq].M); + print_double (secfilt[seq].MpsfChp); } break; @@ -1031,5 +1034,5 @@ print_double (NAN); } else { - print_double (secfilt[seq].dM); + print_double (secfilt[seq].dMpsfChp); } break; @@ -1039,5 +1042,5 @@ print_double (NAN); } else { - print_double (secfilt[seq].Map); + print_double (secfilt[seq].MapChp); } break; @@ -1047,5 +1050,5 @@ print_double (NAN); } else { - print_double (secfilt[seq].dMap); + print_double (secfilt[seq].dMapChp); } break; @@ -1055,5 +1058,5 @@ print_double (NAN); } else { - print_double_exp (secfilt[seq].sMap); + print_double_exp (secfilt[seq].sMapChp); } break; @@ -1063,5 +1066,5 @@ print_double (NAN); } else { - print_double_exp (secfilt[seq].dMap); + print_double_exp (secfilt[seq].dMapChp); } break; @@ -1071,5 +1074,5 @@ print_double (NAN); } else { - print_double (secfilt[seq].Mkron); + print_double (secfilt[seq].MkronChp); } break; @@ -1079,5 +1082,5 @@ print_double (NAN); } else { - print_double (secfilt[seq].dMkron); + print_double (secfilt[seq].dMkronChp); } break; @@ -1087,5 +1090,5 @@ print_double (NAN); } else { - print_double_exp (secfilt[seq].sMkron); + print_double_exp (secfilt[seq].sMkronChp); } break; @@ -1095,5 +1098,5 @@ print_double (NAN); } else { - print_double_exp (secfilt[seq].dMkron); + print_double_exp (secfilt[seq].dMkronChp); } break; Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/images.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/images.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/images.c (revision 40477) @@ -352,6 +352,6 @@ Xvec.Nelements = Yvec.Nelements = N; if (N > 0) { - graphmode.style = 2; /* points */ - graphmode.ptype = 100; /* connect pairs of points */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ + graphmode.ptype = KAPA_POINT_PAIR_CONNECT; /* connect pairs of points */ graphmode.etype = 0; PlotVectorPair (kapa, &Xvec, &Yvec, NULL, &graphmode); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imbox.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imbox.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imbox.c (revision 40477) @@ -156,6 +156,6 @@ Xvec.Nelements = Yvec.Nelements = Npts; if (Npts > 0) { - graphmode.style = 2; /* points */ - graphmode.ptype = 100; /* connect pairs of points */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ + graphmode.ptype = KAPA_POINT_PAIR_CONNECT; /* connect pairs of points */ graphmode.etype = 0; PlotVectorPair (kapa, &Xvec, &Yvec, NULL, &graphmode); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imdata.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imdata.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imdata.c (revision 40477) @@ -183,5 +183,5 @@ for (i = 0; i < catalog.Nmeasure; i++) { if ((catalog.measure[i].t < start) || (catalog.measure[i].t > stop)) continue; - vec[0].elements.Flt[N] = catalog.measure[i].Mcal; + vec[0].elements.Flt[N] = catalog.measure[i].McalPSF; N++; CHECK_REALLOCATE (vec[0].elements.Flt, opihi_flt, NPTS, N, 1000); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imdense.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imdense.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imdense.c (revision 40477) @@ -59,5 +59,5 @@ Xvec.Nelements = Yvec.Nelements = N; if (N > 0) { - graphmode.style = 2; /* points */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ graphmode.etype = 0; PlotVectorPair (kapa, &Xvec, &Yvec, NULL, &graphmode); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imlist.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imlist.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imlist.c (revision 40477) @@ -141,5 +141,5 @@ if (VERBOSE) { gprint (GP_LOG, "%3lld %s %8lld %8.4f %8.4f %f %5d %2d %4.2f %5.3f %5.3f", - (long long) i, image[i].name, (long long) image[i].imageID, r, d, t, image[i].nstar, image[i].photcode, image[i].secz, image[i].Mcal, image[i].dMcal); + (long long) i, image[i].name, (long long) image[i].imageID, r, d, t, image[i].nstar, image[i].photcode, image[i].secz, image[i].McalPSF, image[i].dMcal); if (showUR) { Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imphot.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imphot.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imphot.c (revision 40477) @@ -64,5 +64,5 @@ for (x = 0; x < 100; x+=1.0, p++) { // *p = applyMcal (&image[subset[0]], (fx*x), (fy*y)); - *p = image[subset[0]].Mcal; + *p = image[subset[0]].McalPSF; } } @@ -71,5 +71,5 @@ for (j = 0; j < Nsubset; j++) { i = subset[j]; - gprint (GP_ERR, "%s: %f\n", image[i].name, image[i].Mcal); + gprint (GP_ERR, "%s: %f\n", image[i].name, image[i].McalPSF); // XXX old code when we had the option of a 2D zero point model @@ -77,18 +77,18 @@ switch (image[i].order) { case 0: - gprint (GP_ERR, "%s: %d - %f\n", image[i].name, image[i].order, image[i].Mcal); + gprint (GP_ERR, "%s: %d - %f\n", image[i].name, image[i].order, image[i].McalPSF); break; case 1: - gprint (GP_ERR, "%s: %d - %f, %d %d\n", image[i].name, image[i].order, image[i].Mcal, image[i].Mx, image[i].My); + gprint (GP_ERR, "%s: %d - %f, %d %d\n", image[i].name, image[i].order, image[i].McalPSF, image[i].Mx, image[i].My); break; case 2: - gprint (GP_ERR, "%s: %d - %f, %d %d, %d %d %d\n", image[i].name, image[i].order, image[i].Mcal, image[i].Mx, image[i].My, image[i].Mxx, image[i].Mxy, image[i].Myy); + gprint (GP_ERR, "%s: %d - %f, %d %d, %d %d %d\n", image[i].name, image[i].order, image[i].McalPSF, image[i].Mx, image[i].My, image[i].Mxx, image[i].Mxy, image[i].Myy); break; case 3: - gprint (GP_ERR, "%s: %d - %f, %d %d, %d %d %d, %d %d %d %d\n", image[i].name, image[i].order, image[i].Mcal, image[i].Mx, image[i].My, + gprint (GP_ERR, "%s: %d - %f, %d %d, %d %d %d, %d %d %d %d\n", image[i].name, image[i].order, image[i].McalPSF, image[i].Mx, image[i].My, image[i].Mxx, image[i].Mxy, image[i].Myy, image[i].Mxxx, image[i].Mxxy, image[i].Mxyy, image[i].Myyy); break; case 4: - gprint (GP_ERR, "%s: %d - %f, %d %d, %d %d %d, %d %d %d %d, %d %d %d %d %d\n", image[i].name, image[i].order, image[i].Mcal, image[i].Mx, image[i].My, + gprint (GP_ERR, "%s: %d - %f, %d %d, %d %d %d, %d %d %d %d, %d %d %d %d %d\n", image[i].name, image[i].order, image[i].McalPSF, image[i].Mx, image[i].My, image[i].Mxx, image[i].Mxy, image[i].Myy, image[i].Mxxx, image[i].Mxxy, image[i].Mxyy, image[i].Myyy, image[i].Mxxxx, image[i].Mxxxy, image[i].Mxxyy, image[i].Mxyyy, image[i].Myyyy); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imstats.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imstats.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/imstats.c (revision 40477) @@ -41,5 +41,5 @@ Xvec.elements.Flt[i] = image[i].secz; if (Mcal) - Yvec.elements.Flt[i] = image[i].Mcal; + Yvec.elements.Flt[i] = image[i].McalPSF; else Yvec.elements.Flt[i] = image[i].dMcal; @@ -47,9 +47,9 @@ gprint (GP_ERR, "%d %8.4f %8.4f %10d %6d %5.3f %6.3f %6.3f\n", i, r, d, image[i].tzero, image[i].nstar, Xvec.elements.Flt[i], - image[i].Mcal, image[i].dMcal); + image[i].McalPSF, image[i].dMcal); } if (AutoLimits) SetLimits (&Xvec, &Yvec, &graphmode); - graphmode.style = 2; + graphmode.style = KAPA_PLOT_POINTS; /* points */ graphmode.etype = 0; PlotVectorPair (kapa, &Xvec, &Yvec, NULL, &graphmode); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/init.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/init.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/init.c (revision 40477) @@ -3,4 +3,6 @@ int avextract PROTO((int, char **)); int avmatch PROTO((int, char **)); +int avperiodogram PROTO((int, char **)); +int avperiodomatch PROTO((int, char **)); int badimages PROTO((int, char **)); int calextract PROTO((int, char **)); @@ -67,4 +69,6 @@ {1, "avextract", avextract, "extract average data values"}, {1, "avmatch", avmatch, "extract average data values matched to RA,DEC points"}, + {1, "avperiodogram", avperiodogram, "perform periodogram on objects based on restrictions"}, + {1, "avperiodomatch", avperiodomatch, "perform periodogram on objects based on ra,dec list"}, {1, "badimages", badimages, "look for images with anomalous astrometry"}, // {1, "calextract", calextract, "extract photometry calibration"}, Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/objectcoverage.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/objectcoverage.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/objectcoverage.c (revision 40477) @@ -203,5 +203,5 @@ if (catalog.secfilt[j*Nsecfilt+Nsec].Ncode < 2) { continue; } - invalid = ((catalog.secfilt[j*Nsecfilt + Nsec].M < 1.0) || (isnan(catalog.secfilt[j*Nsecfilt + Nsec].M))); + invalid = ((catalog.secfilt[j*Nsecfilt + Nsec].MpsfChp < 1.0) || (isnan(catalog.secfilt[j*Nsecfilt + Nsec].MpsfChp))); if (invalid) continue; Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/paverage.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/paverage.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/paverage.c (revision 40477) @@ -81,5 +81,5 @@ return (FALSE); } - graphmode.style = 2; /* set style to points */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ graphmode.size = -1; /* point size determined by Zvec */ graphmode.etype = 0; /* no errorbars */ @@ -125,5 +125,5 @@ while (average[i].R > Rmax) average[i].R -= 360.0; - mag = secfilt[i*Nsecfilt+Nsec].M; + mag = secfilt[i*Nsecfilt+Nsec].MpsfChp; Zvec[Npts] = MIN (1.0, MAX (0.01, (mag - Mz) / Mr)); if (LimExclude && (Zvec[Npts] > 0.99)) continue; Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/pmeasure.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/pmeasure.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/pmeasure.c (revision 40477) @@ -154,5 +154,5 @@ return (FALSE); } - graphmode.style = 2; /* set style to points */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ graphmode.size = -1; /* point size determined by Zvec */ graphmode.etype = 0; /* no errorbars */ Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/procks.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/procks.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/procks.c (revision 40477) @@ -111,5 +111,5 @@ Yvec.Nelements = Xvec.Nelements = N; - graphmode.style = 2; /* set style to points */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ graphmode.etype = 0; /* no errorbars */ PlotVectorPair (kapa, &Xvec, &Yvec, NULL, &graphmode); @@ -142,6 +142,6 @@ Yvec.Nelements = Xvec.Nelements = N; - graphmode.style = 2; /* set style to points */ - graphmode.ptype = 100; /* connect pairs */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ + graphmode.ptype = KAPA_POINT_PAIR_CONNECT; /* connect pairs of points */ graphmode.etype = 0; /* no errorbars */ Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/region_list.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/region_list.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/region_list.c (revision 40477) @@ -181,4 +181,16 @@ SkyList *skylist; + // the list of regions comes directly from a file + if (selection->list != NULL) { + skylist = SkyListLoadFile (selection->list); + return (skylist); + } + + // all other options require sky to be set + if (!sky) { + gprint (GP_ERR, "CATDIR not set\n"); + return NULL; + } + /* determine region-file names */ if (selection->name != NULL) { @@ -186,9 +198,4 @@ return (skylist); } - - if (selection->list != NULL) { - skylist = SkyListLoadFile (selection->list); - return (skylist); - } if (selection->useDisplay) { Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/remote.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/remote.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/remote.c (revision 40477) @@ -27,5 +27,8 @@ gprint (GP_ERR, " -skip-result : do not try to read from the result file\n"); gprint (GP_ERR, "OR: remote -reload (uniquer)\n"); + gprint (GP_ERR, " (reloads the remote host results into vectors as if a parallel command were run)\n"); gprint (GP_ERR, "OR: remote -get-results (uniquer)\n"); + gprint (GP_ERR, " (generates the list of remote result filenames and status variables)\n"); + gprint (GP_ERR, " (RESULT_FILE:i is the filenme, RESULT_STATUS:i is the dvo_client exit status)\n"); return FALSE; } Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/showtile.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/showtile.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/showtile.c (revision 40477) @@ -78,6 +78,6 @@ Xvec.Nelements = Yvec.Nelements = N; if (N > 0) { - graphmode.style = 2; /* points */ - graphmode.ptype = 100; /* connect pairs of points */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ + graphmode.ptype = KAPA_POINT_PAIR_CONNECT; /* connect pairs of points */ graphmode.etype = 0; PlotVectorPair (kapa, &Xvec, &Yvec, NULL, &graphmode); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/simage.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/simage.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/simage.c (revision 40477) @@ -118,5 +118,5 @@ } - graphmode.style = 2; + graphmode.style = KAPA_PLOT_POINTS; graphmode.size = -1; graphmode.etype = 0; Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/skycat.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/skycat.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/skycat.c (revision 40477) @@ -121,6 +121,6 @@ Xvec.Nelements = Yvec.Nelements = Npts; if (Npts > 0) { - graphmode.style = 2; /* points */ - graphmode.ptype = 100; /* connect pairs of points */ + graphmode.style = KAPA_PLOT_POINTS; /* points */ + graphmode.ptype = KAPA_POINT_PAIR_CONNECT; /* connect pairs of points */ graphmode.etype = 0; PlotVectorPair (kapa, &Xvec, &Yvec, NULL, &graphmode); Index: /branches/czw_branch/20170908/Ohana/src/opihi/dvo/skycoverage.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/dvo/skycoverage.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/dvo/skycoverage.c (revision 40477) @@ -330,8 +330,8 @@ break; case MIN_MCAL: - V[ys*Nx + xs] = MIN(V[ys*Nx + xs], image[i].Mcal); + V[ys*Nx + xs] = MIN(V[ys*Nx + xs], image[i].McalPSF); break; case MAX_MCAL: - V[ys*Nx + xs] = MAX(V[ys*Nx + xs], image[i].Mcal); + V[ys*Nx + xs] = MAX(V[ys*Nx + xs], image[i].McalPSF); break; case MIN_TIME: { Index: /branches/czw_branch/20170908/Ohana/src/opihi/include/astro.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/include/astro.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/include/astro.h (revision 40477) @@ -47,7 +47,7 @@ double VectorFractionInterpolate (double *values, float fraction, int Npts); -int PlxSetMeanEpoch (double *R, double *D, double *T, double *Rmean, double *Dmean, double *Tmean, int *mask, int Ntotal); -int PlxSetEpochPosition (PlxFitData *fitdata, double *R, double *D, double *dR, double *dD, double *T, int *mask, int Ntotal, Coords *coords, double Tmean); -int PlxOutlierClip (PlxFitData *fitdata, int *mask, int Noutlier, float dPsigMax, Vector *dPvec, int VERBOSE); +int PlxSetMeanEpoch (double *R, double *D, double *T, double *Rmean, double *Dmean, double *Tmean, opihi_int *mask, int Ntotal); +int PlxSetEpochPosition (PlxFitData *fitdata, double *R, double *D, double *dR, double *dD, double *T, opihi_int *mask, int Ntotal, Coords *coords, double Tmean); +int PlxOutlierClip (PlxFitData *fitdata, opihi_int *mask, int Noutlier, float dPsigMax, Vector *dPvec, int VERBOSE); int PlxFitDataAlloc (PlxFitData *data, int N); Index: /branches/czw_branch/20170908/Ohana/src/opihi/include/data.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/include/data.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/include/data.h (revision 40477) @@ -50,4 +50,24 @@ float **buffers; } MedImageType; + +/*** typedef structs used by the Neural Network functions (nnet_*) ***/ +typedef struct { + char *name; + int Nlayer; // Nlayers = input layer + output layer + hidden layers + int *Nnodes; // number of nodes per layer + float **weight; // a matrix between each layer + float **biases; // a vector for each layer + + float **zvalue; // a vector of z values for each layer (= w*input + b) + float **svalue; // a vector of s values for each layer (= sigmoid(z)) + float **sprime; // + float **delta; // + + float ** Nabla_b; // a vector of Nabla_b values for each layer + float **dNabla_b; // a vector of Nabla_b values for each layer + + float ** Nabla_w; // a matrix of Nabla_w values for each layer + float **dNabla_w; // a matrix of Nabla_w values for each layer +} Nnet; void InitData (void); @@ -162,4 +182,6 @@ /* starfuncs.c */ double get_aperture_stats (Matrix *matrix, int X, int Y, int Npix, int Nborder, double max, int VERBOSE); +double get_box_stats (Matrix *matrix, int X, int Y, int dX, int dY, int Nborder, double max, int VERBOSE); + int set_rough_radii (double Ra, double Ri, double Ro); int get_rough_star (float *data, int Nx, int Ny, int x, int y, opihi_flt *xc, opihi_flt *yc, opihi_flt *sx, opihi_flt *sy, opihi_flt *sxy, opihi_flt *zs, opihi_flt *zp, opihi_flt *sk); @@ -227,3 +249,42 @@ int GetKapaChannelFromString (char *string); +/*** Neural Network functions (nnet_*) ***/ + +// in lib.data/nnet.c: +void InitNnets (); +void InitNnetData (Nnet *nnet, char *name, int Nlayer); + +void FreeNnets (); +void FreeNnetData (Nnet *nnet); + +Nnet *GetNnet (int where); +Nnet *FindNnet (char *name); +Nnet *CreateNnet (char *name, int Nlayer); +void CreateNnetData (Nnet *nnet, int LargeWeightInit); + +void PrintNnet (Nnet *nnet); +int DeleteNnet (Nnet *nnet); +void ListNnets (); + +// in cmd.data/nnet.c: +int nnet_command (int argc, char **argv); + +// in cmd.data/nnet_commands.c: +int nnet_init (int argc, char **argv); +int nnet_list (int argc, char **argv); +int nnet_delete (int argc, char **argv); +int nnet_create (int argc, char **argv); +int nnet_set (int argc, char **argv); +int nnet_get (int argc, char **argv); +int nnet_read (int argc, char **argv); +int nnet_write (int argc, char **argv); +int nnet_print (int argc, char **argv); + +// in cmd.data/nnet_train.c: +int nnet_train (int argc, char **argv); +void nnet_feedforward (Nnet *nnet); + +// in cmd.data/nnet_apply.c: +int nnet_apply (int argc, char **argv); + # endif Index: /branches/czw_branch/20170908/Ohana/src/opihi/include/dvoshell.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/include/dvoshell.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/include/dvoshell.h (revision 40477) @@ -41,4 +41,20 @@ char *HOSTDIR; char *RESULT_FILE; + +typedef struct { + int Nmeasure; + int Nperiods; + opihi_flt *period; + opihi_flt *power; + double P50; + double R50; + double R90; +} PeriodogramResult; + +/*** Periodogram functions (avperiodogram and avperiodomatch) ***/ +void PeriodogramResultFree (PeriodogramResult *result); +void PeriodogramResultSave (PeriodogramResult *result, char *extname, FILE *foutput, Average *average); +void PeriodogramSetOptions (float minimumPeriod, float maximumPeriod); +PeriodogramResult *PeriodogramRawFloatingMean (opihi_flt *time, opihi_flt *flux, opihi_flt *dflux, int Npts); /*** dvo prototypes ***/ Index: /branches/czw_branch/20170908/Ohana/src/opihi/include/pantasks.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/include/pantasks.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/include/pantasks.h (revision 40477) @@ -8,4 +8,6 @@ # define DEBUG 0 + +typedef int IDtype; typedef enum { @@ -133,5 +135,5 @@ typedef struct { - int JobID; /* internal ID for job */ + IDtype JobID; /* internal ID for job */ int pid; /* external ID for job */ @@ -204,8 +206,7 @@ void UpdateTaskTimerStats (Task *task, int mode, double dtime); -int NextJobID (void); +IDtype NextJobID (void); void InitJobIDs (void); void FreeJobIDs (void); -int FreeJobID (int ID); void InitJobs (void); @@ -213,5 +214,5 @@ Job *NextJob (void); -Job *FindJob (int JobID); +Job *FindJob (IDtype JobID); void ListJobs (void); Job *CreateJob (Task *task); @@ -234,5 +235,5 @@ int SubmitControllerJob (Job *job); int DeleteControllerJob (Job *job); -Job *FindControllerJob (int JobID); +Job *FindControllerJob (IDtype JobID); int StartController (void); int ControllerCommand (char *command, char *response, IOBuffer *buffer); Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/Makefile =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/Makefile (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/Makefile (revision 40477) @@ -20,5 +20,5 @@ $(SDIR)/fft.$(ARCH).o \ $(SDIR)/svdcmp.$(ARCH).o \ -$(SDIR)/svdcmp_bond_new.$(ARCH).o \ +$(SDIR)/svdcmp_bond_new.$(ARCH).o \ $(SDIR)/bracket.$(ARCH).o \ $(SDIR)/spline.$(ARCH).o \ @@ -27,4 +27,5 @@ $(SDIR)/mrqmin.$(ARCH).o \ $(SDIR)/mrq2dmin.$(ARCH).o \ +$(SDIR)/nnet.$(ARCH).o \ $(SDIR)/precess.$(ARCH).o \ $(SDIR)/starfuncs.$(ARCH).o \ Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/gaussian.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/gaussian.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/gaussian.c (revision 40477) @@ -26,6 +26,6 @@ if (Ngaussint == Nbin) return; - long A = time(NULL); - srand48(A); + // long A = time(NULL); + // srand48() is called by startup Ngaussint = Nbin; Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/graphtools.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/graphtools.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/graphtools.c (revision 40477) @@ -10,5 +10,5 @@ if (xvec != NULL) { if (xvec->type == OPIHI_FLT) { - maxX = DBL_MIN; + maxX = -DBL_MAX; minX = DBL_MAX; for (i = 0; i < xvec[0].Nelements; i++) { @@ -33,5 +33,5 @@ if (yvec != NULL) { if (yvec->type == OPIHI_FLT) { - maxY = DBL_MIN; + maxY = -DBL_MAX; minY = DBL_MAX; for (i = 0; i < yvec[0].Nelements; i++) { Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/nnet.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/nnet.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/nnet.c (revision 40477) @@ -0,0 +1,253 @@ +# include "data.h" + +static Nnet **nnets = NULL; /* nnet to store the list of all nnets */ +static int Nnnets = 0; /* number of currently defined nnets */ +static int NNNETS = 0; /* number of currently allocated nnets */ + +void InitNnets () { + Nnnets = 0; + NNNETS = 16; + ALLOCATE (nnets, Nnet *, NNNETS); +} + +void FreeNnets () { + + int i; + + if (!nnets) InitNnets(); + + for (i = 0; i < Nnnets; i++) { + FreeNnetData (nnets[i]); + free (nnets[i]); + } + free (nnets); +} + +// InitNnetData requires nnet to unassigned or initialized +// if you have an existing Nnet, call FreeNnetData first +void InitNnetData (Nnet *nnet, char *name, int Nlayer) { + + nnet[0].name = strcreate (name); + + nnet[0].Nlayer = 0; + nnet[0].Nnodes = NULL; + nnet[0].weight = NULL; + nnet[0].biases = NULL; + nnet[0].svalue = NULL; + nnet[0].zvalue = NULL; + nnet[0].sprime = NULL; + nnet[0].delta = NULL; + + nnet[0]. Nabla_b = NULL; + nnet[0].dNabla_b = NULL; + nnet[0]. Nabla_w = NULL; + nnet[0].dNabla_w = NULL; + + nnet[0].Nlayer = Nlayer; + ALLOCATE_ZERO (nnet[0].Nnodes, int, Nlayer); + ALLOCATE_ZERO (nnet[0].weight, float *, Nlayer); + ALLOCATE_ZERO (nnet[0].biases, float *, Nlayer); + ALLOCATE_ZERO (nnet[0].svalue, float *, Nlayer); + ALLOCATE_ZERO (nnet[0].zvalue, float *, Nlayer); + ALLOCATE_ZERO (nnet[0].sprime, float *, Nlayer); + ALLOCATE_ZERO (nnet[0].delta , float *, Nlayer); + + ALLOCATE_ZERO (nnet[0]. Nabla_b, float *, Nlayer); + ALLOCATE_ZERO (nnet[0].dNabla_b, float *, Nlayer); + ALLOCATE_ZERO (nnet[0]. Nabla_w, float *, Nlayer); + ALLOCATE_ZERO (nnet[0].dNabla_w, float *, Nlayer); +} + +void FreeNnetData (Nnet *nnet) { + + int i; + + FREE (nnet[0].name); + for (i = 0; i < nnet[0].Nlayer; i++) { + FREE (nnet[0].weight[i]); + FREE (nnet[0].biases[i]); + FREE (nnet[0].svalue[i]); + FREE (nnet[0].zvalue[i]); + FREE (nnet[0].sprime[i]); + FREE (nnet[0].delta [i]); + + FREE (nnet[0]. Nabla_b[i]); + FREE (nnet[0].dNabla_b[i]); + FREE (nnet[0]. Nabla_w[i]); + FREE (nnet[0].dNabla_w[i]); + } + FREE (nnet[0].Nnodes); + FREE (nnet[0].weight); + FREE (nnet[0].biases); + FREE (nnet[0].svalue); + FREE (nnet[0].zvalue); + FREE (nnet[0].sprime); + FREE (nnet[0].delta ); + + FREE (nnet[0]. Nabla_b); + FREE (nnet[0].dNabla_b); + FREE (nnet[0]. Nabla_w); + FREE (nnet[0].dNabla_w); +} + +/* return the given nnet */ +Nnet *GetNnet (int where) { + + if (!nnets) InitNnets(); + + if (where < 0) where += Nnnets; + if (where < 0) return NULL; + if (where >= Nnnets) return NULL; + return (nnets[where]); +} + +/* return the given nnet */ +Nnet *FindNnet (char *name) { + + int i; + + if (!nnets) InitNnets(); + + for (i = 0; i < Nnnets; i++) { + if (!strcmp (nnets[i][0].name, name)) { + return (nnets[i]); + } + } + return (NULL); +} + +/* make a new named nnet with Nlayer */ +Nnet *CreateNnet (char *name, int Nlayer) { + + int N; + Nnet *nnet; + + if (!nnets) InitNnets(); + + nnet = FindNnet (name); + if (nnet != NULL) { + FreeNnetData (nnet); + InitNnetData (nnet, name, Nlayer); + return (nnet); + } + + N = Nnnets; + Nnnets ++; + CHECK_REALLOCATE (nnets, Nnet *, NNNETS, Nnnets, 16); + ALLOCATE (nnet, Nnet, 1); + InitNnetData (nnet, name, Nlayer); + + // save this nnet in the static array of nnets + nnets[N] = nnet; + return (nnet); +} + +void CreateNnetData (Nnet *nnet, int LargeWeightInit) { + + ohana_gaussdev_init (); + + // NOTE : none of these elements are used for the first layer (the input layer) + // EXCEPT svalue[0] + + nnet[0].weight[0] = NULL; + nnet[0].biases[0] = NULL; + nnet[0].zvalue[0] = NULL; + nnet[0].sprime[0] = NULL; + nnet[0].delta [0] = NULL; + nnet[0]. Nabla_b[0] = NULL; + nnet[0].dNabla_b[0] = NULL; + nnet[0]. Nabla_w[0] = NULL; + nnet[0].dNabla_w[0] = NULL; + + ALLOCATE (nnet[0].svalue[0], float, nnet[0].Nnodes[0]); + + for (int i = 1; i < nnet[0].Nlayer; i++) { + ALLOCATE (nnet[0].biases[i], float, nnet[0].Nnodes[i]); // biases for each node in the hidden and output layers only + for (int j = 0; j < nnet[0].Nnodes[i]; j++) { + nnet[0].biases[i][j] = ohana_gaussdev_rnd (0.0, 1.0); + } + + float sigma = LargeWeightInit ? 1.0 : 1.0 / sqrt(nnet[0].Nnodes[i-1]); + + ALLOCATE (nnet[0].weight[i], float, nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]); // weight connected each node in the previous layer to the current layer (excludes input layer) + for (int j = 0; j < nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]; j++) { + nnet[0].weight[i][j] = ohana_gaussdev_rnd (0.0, sigma); + } + + ALLOCATE (nnet[0].svalue[i], float, nnet[0].Nnodes[i]); // vectors for holding results / values for each node in the input, hidden, output layers + ALLOCATE (nnet[0].zvalue[i], float, nnet[0].Nnodes[i]); // vectors for holding results / values for each node in the input, hidden, output layers + ALLOCATE (nnet[0].sprime[i], float, nnet[0].Nnodes[i]); // vectors for holding results / values for each node in the input, hidden, output layers + ALLOCATE (nnet[0].delta [i], float, nnet[0].Nnodes[i]); // vectors for holding results / values for each node in the input, hidden, output layers + + ALLOCATE (nnet[0]. Nabla_b[i], float, nnet[0].Nnodes[i]); + ALLOCATE (nnet[0].dNabla_b[i], float, nnet[0].Nnodes[i]); + ALLOCATE (nnet[0]. Nabla_w[i], float, nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]); + ALLOCATE (nnet[0].dNabla_w[i], float, nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]); + } +} + +/* delete a nnet */ +int DeleteNnet (Nnet *nnet) { + + int i, N, NNNETS_2; + + if (!nnets) InitNnets(); + + /* find nnet in nnet list */ + N = -1; + for (i = 0; i < Nnnets; i++) { + if (nnets[i] == nnet) { + N = i; + break; + } + } + if (N == -1) return (FALSE); + + for (i = N; i < Nnnets - 1; i++) { + nnets[i] = nnets[i + 1]; + } + Nnnets --; + NNNETS_2 = MAX (16, NNNETS / 2); + if (Nnnets < NNNETS_2) { + NNNETS = NNNETS_2; + REALLOCATE (nnets, Nnet *, NNNETS); + } + + FreeNnetData (nnet); + free (nnet); + return (TRUE); +} + +/* list known nnets */ +void ListNnets () { + + int i, j; + + if (!nnets) InitNnets(); + + for (i = 0; i < Nnnets; i++) { + gprint (GP_ERR, "%-15s :", nnets[i][0].name); + for (j = 0; j < nnets[i][0].Nlayer; j++) { + gprint (GP_ERR, " %3d", nnets[i][0].Nnodes[j]); + } + gprint (GP_ERR, "\n"); + } + return; +} + +void PrintNnet (Nnet *nnet) { + + for (int L = 1; L < nnet[0].Nlayer; L++) { + gprint (GP_ERR, " ----- Layer %d -----\n", L); + for (int j = 0; j < nnet[0].Nnodes[L]; j++) { + for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) { + int k = j * nnet[0].Nnodes[L-1] + i; + myAssert (k < nnet[0].Nnodes[L-1]*nnet[0].Nnodes[L], "overflow"); + gprint (GP_ERR, "%7.4f ", nnet[0].weight[L][k]); + } + gprint (GP_ERR, " : %7.4f\n", nnet[0].biases[L][j]); + } + } + return; +} + Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/starfuncs.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/starfuncs.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.data/starfuncs.c (revision 40477) @@ -101,4 +101,118 @@ } +double get_box_stats (Matrix *matrix, int X, int Y, int dX, int dY, int Nborder, double max, int VERBOSE) { + + double *ring; + double x, y, x2, y2, xy, I, sky, FWHMx, FWHMy, value, mag, Sxy; + int i, j, n, Nring, Nmax; + double Npts, gain, dsky2, dmag, peak, offset; + char *string; + + string = get_variable ("GAIN"); + if (string == (char *) NULL) { + gprint (GP_ERR, "assuming a value of 1.0\n"); + gain = 1.0; + } else { + gain = atof (string); + } + Nborder = MAX (1, Nborder); + Nborder = MIN (1000, Nborder); + + int dX2 = (int)(0.5*dX); + int dY2 = (int)(0.5*dY); + dX = 2 * dX2 + 1; + dY = 2 * dY2 + 1; + + Nring = 2*Nborder*(dX + 2*Nborder) + 2*Nborder*(dY + 2*Nborder); + ALLOCATE (ring, double, Nring); + bzero (ring, sizeof(double)*Nring); + + // get the pixels in the border regions: + // XXX gfits_get_matrix_value returns 0 for out-of-bounds pixels, but should return NAN + // and they should be skipped + n = 0; + for (j = 0; j < Nborder; j++) { + for (i = X - dX2 - Nborder; i < X + dX2 + Nborder + 1; i++) { + value = gfits_get_matrix_value (matrix, i, (int)(Y - dY2 - j)); + if (isfinite(value)) { ring[n] = value; n++; } + value = gfits_get_matrix_value (matrix, i, (int)(Y + dY2 + j)); + if (isfinite(value)) { ring[n] = value; n++; } + } + for (i = Y - dY2; i < Y + dY2 + 1; i++) { + value = gfits_get_matrix_value (matrix, (int)(X - dX2 - j), i); + if (isfinite(value)) { ring[n] = value; n++; } + value = gfits_get_matrix_value (matrix, (int)(X + dX2 + j), i); + if (isfinite(value)) { ring[n] = value; n++; } + } + } + Nring = n; + dsort (ring, Nring); + for (Npts = sky = dsky2 = 0, i = 0.25*Nring; i < 0.75*Nring; i++, Npts += 1.0) { + sky += ring[i]; + dsky2 += ring[i]*ring[i]; + } + sky = sky / Npts; + dsky2 = dsky2 / Npts - sky*sky; + free (ring); + + float dx, dy; + + peak = 0; + Npts = Nmax = 0; + x = y = x2 = y2 = xy = I = 0; + for (i = X - dX2; i < X + dX2 + 1; i++) { + for (j = Y - dY2; j < Y + dY2 + 1; j++) { + value = gfits_get_matrix_value (matrix, i, j); + if (!isfinite(value)) continue; + offset = value - sky; + dx = i - X; + dy = j - Y; + x += dx*offset; + y += dy*offset; + x2 += dx*dx*offset; + y2 += dy*dy*offset; + xy += dx*dy*offset; + I += offset; + Npts ++; + if (value > max) { + Nmax ++; + } + if (value > peak) peak = value; + } + } + + x = x / I; + y = y / I; + FWHMx = 2.355*sqrt (fabs(x2 / I - x*x)); + FWHMy = 2.355*sqrt (fabs(y2 / I - y*y)); + Sxy = xy / I - x*y; + mag = -2.5*log10(I); + + // flux_error = sqrt( I + Npts*dsky2 ) + // dmag = 1.086 * flux_error / flux + dmag = 1.086 * sqrt (fabs(I + Npts*dsky2)) / (gain * I); + x = x + X; + y = y + Y; + + set_variable ("Xg", x); + set_variable ("Yg", y); + set_variable ("SXg", FWHMx); + set_variable ("SYg", FWHMy); + set_variable ("SXYg", Sxy); + set_variable ("Sg", sky); + set_variable ("dSg", sqrt (fabs (dsky2))); + set_variable ("Zg", mag); + set_variable ("dZg", dmag); + set_variable ("Zcg", I); + set_variable ("Zpk", peak); + set_int_variable ("Nsat", Nmax); + set_int_variable ("Npts", Npts); + + if (VERBOSE) gprint (GP_LOG, "%f %f %f %f %f %f %f %f\n", x, y, FWHMx, FWHMy, sky, I, mag, dmag); + + return (mag); + +} + static double Raper = 5; static double Rinner = 10; @@ -171,4 +285,5 @@ off = j*Nx; for (i = Xs; i < Xe; i++) { + if (!isfinite(data[i+off])) continue; sky[Nsky] = data[i+off]; Nsky ++; @@ -180,4 +295,5 @@ off = j*Nx; for (i = Xs; i < Xe; i++) { + if (!isfinite(data[i+off])) continue; sky[Nsky] = data[i+off]; Nsky ++; @@ -191,4 +307,5 @@ off = j*Nx; for (i = Xs; i < Xe; i++) { + if (!isfinite(data[i+off])) continue; sky[Nsky] = data[i+off]; Nsky ++; @@ -200,4 +317,5 @@ off = j*Nx; for (i = Xs; i < Xe; i++) { + if (!isfinite(data[i+off])) continue; sky[Nsky] = data[i+off]; Nsky ++; @@ -227,4 +345,5 @@ rad2 = SQ(Xc) + SQ(Yc); if (rad2 > Ro2) continue; + // if (!isfinite(data[i+off])) continue; value = data[i+off] - fsky; Sx += Xc*value; Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/VectorIO.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/VectorIO.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/VectorIO.c (revision 40477) @@ -41,5 +41,5 @@ for (j = 0; j < Nvec; j++) { // if the format is not defined, just use the native byte-widths - tformat[2*j + 0] = (vec[j][0].type == OPIHI_FLT) ? 'D' : 'J'; + tformat[2*j + 0] = (vec[j][0].type == OPIHI_FLT) ? 'D' : 'K'; // this depends on opihi_int == int64_t for Int tformat[2*j + 1] = 0; } @@ -60,7 +60,8 @@ for (j = 0; j < Nvec; j++) { if (vec[j][0].type == OPIHI_FLT) { - gfits_set_bintable_column_reformat (theader, ftable, vec[j][0].name, "double", vec[j][0].elements.Flt, vec[j][0].Nelements, nativeOrder); + gfits_set_bintable_column_reformat (theader, ftable, vec[j][0].name, "double", vec[j][0].elements.Flt, vec[j][0].Nelements, nativeOrder); } else { - gfits_set_bintable_column_reformat (theader, ftable, vec[j][0].name, "int", vec[j][0].elements.Int, vec[j][0].Nelements, nativeOrder); +// gfits_set_bintable_column_reformat (theader, ftable, vec[j][0].name, "int", vec[j][0].elements.Int, vec[j][0].Nelements, nativeOrder); + gfits_set_bintable_column_reformat (theader, ftable, vec[j][0].name, "int64_t", vec[j][0].elements.Int, vec[j][0].Nelements, nativeOrder); } } @@ -328,5 +329,6 @@ ASSIGN_DATA(short, short, Int); ASSIGN_DATA(int, int, Int); - ASSIGN_DATA(int64_t, int64_t, Flt); // int64_t has a problem: Int is too small, Flt is wrong precision + ASSIGN_DATA(int64_t, int64_t, Int); // XXX this works if opihi_int is assigned to int64_t +//ASSIGN_DATA(int64_t, int64_t, Flt); // int64_t has a problem: Int is too small, Flt is wrong precision ASSIGN_DATA(float, float, Flt); ASSIGN_DATA(double, double, Flt); @@ -353,5 +355,6 @@ ASSIGN_DATA_TRANSPOSE(short, short, Int); ASSIGN_DATA_TRANSPOSE(int, int, Int); - ASSIGN_DATA_TRANSPOSE(int64_t, int64_t, Flt); + ASSIGN_DATA_TRANSPOSE(int64_t, int64_t, Int); +//ASSIGN_DATA_TRANSPOSE(int64_t, int64_t, Flt); // see above comment ASSIGN_DATA_TRANSPOSE(float, float, Flt); ASSIGN_DATA_TRANSPOSE(double, double, Flt); Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/check_stack.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/check_stack.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/check_stack.c (revision 40477) @@ -48,6 +48,6 @@ Nz = stack[i].buffer[0].matrix.Naxis[2]; } - if ((Nx != stack[i].buffer[0].matrix.Naxis[0]) && - (Ny != stack[i].buffer[0].matrix.Naxis[1]) && + if ((Nx != stack[i].buffer[0].matrix.Naxis[0]) || + (Ny != stack[i].buffer[0].matrix.Naxis[1]) | (Nz != stack[i].buffer[0].matrix.Naxis[2])) { push_error ("dimensions don't match"); Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/convert_to_RPN.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/convert_to_RPN.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/convert_to_RPN.c (revision 40477) @@ -120,5 +120,4 @@ Nop_stack ++; break; - case ST_UNARY: case ST_BINARY: case ST_TRINARY: @@ -139,4 +138,5 @@ Nop_stack ++; break; + case ST_UNARY: case ST_LEFT: /* push operator on OP stack */ Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/dvomath.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/dvomath.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/dvomath.c (revision 40477) @@ -29,4 +29,8 @@ unsigned int Ncstack; cstack = isolate_elements (argc, argv, &Ncstack); + + // for (i = 0; i < Ncstack; i++) { + // fprintf (stderr, "%d : %s\n", i, cstack[i]); + // } /* generate RPN stack from cstack arguments */ @@ -82,5 +86,5 @@ } else { if (stack[0].type == ST_SCALAR_INT) { - sprintf (outname, "%d", stack[0].IntValue); + sprintf (outname, OPIHI_INT_FMT, stack[0].IntValue); } else { sprintf (outname, "%.12g", stack[0].FltValue); Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/evaluate_stack.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/evaluate_stack.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/evaluate_stack.c (revision 40477) @@ -89,5 +89,5 @@ } if (tmp_stack.type == ST_SCALAR_INT) { - gprint (GP_ERR, "---> %d ", tmp_stack.IntValue); + gprint (GP_ERR, "---> "OPIHI_INT_FMT" ", tmp_stack.IntValue); } if (tmp_stack.type == ST_SCALAR_FLT) { @@ -110,5 +110,5 @@ if (i < 3) { /* need two variables to operate on */ - snprintf (line, 512, "syntax error: trinary operator without three operands: %s\n", stack[i].name); + snprintf (line, 512, "syntax error: trinary operator without three operands: %s\n(Note that the : in a trinary operation must be protected by spaces", stack[i].name); push_error (line); clear_stack (&tmp_stack); @@ -124,5 +124,5 @@ /* there are no valid unary string operators */ - snprintf (line, 512, "invalid operands for trinary operator %s (mismatch types?)", stack[i].name); + snprintf (line, 512, "invalid operands for trinary operator %s (mismatch types?)\n(Note that the : in a trinary operation must be protected by spaces)", stack[i].name); push_error (line); clear_stack (&tmp_stack); @@ -131,5 +131,5 @@ got_three_op: if (!status) { - snprintf (line, 512, "syntax error: invalid operand for trinary operation: %s or %s or %s\n", stack[i-1].name, stack[i-2].name, stack[i-3].name); + snprintf (line, 512, "syntax error: invalid operand for trinary operation: %s or %s or %s\n(Note that the : in a trinary operation must be protected by spaces)", stack[i-1].name, stack[i-2].name, stack[i-3].name); push_error (line); clear_stack (&tmp_stack); @@ -186,6 +186,26 @@ got_two_op: if (!status) { - snprintf (line, 512, "syntax error: invalid operand for binary operation: %s or %s\n", stack[i-1].name, stack[i-2].name); + // we are guaranteed to have stack[i-1] and stack[i-2] since i >= 2 (above) + char tmpvector[] = "Temporary Vector"; + char tmpmatrix[] = "Temporary Matrix"; + + char *name1 = NULL; + if (stack[i-1].name) { + name1 = stack[i-1].name; + } else { + if (stack[i-1].type == ST_VECTOR_TMP) name1 = tmpvector; + if (stack[i-1].type == ST_MATRIX_TMP) name1 = tmpmatrix; + } + char *name2 = NULL; + if (stack[i-2].name) { + name2 = stack[i-2].name; + } else { + if (stack[i-2].type == ST_VECTOR_TMP) name2 = tmpvector; + if (stack[i-2].type == ST_MATRIX_TMP) name2 = tmpmatrix; + } + + snprintf (line, 512, "syntax error: invalid operand for binary operation: %s or %s\n", name1, name2); push_error (line); + int isTrinary = TRUE; isTrinary = isTrinary && (i >= 2); @@ -199,4 +219,5 @@ push_error (line); } + clear_stack (&tmp_stack); return (FALSE); Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/parse.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/parse.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/parse.c (revision 40477) @@ -145,8 +145,10 @@ /* simple variable assignment */ /* dvomath returns a new string, or NULL, with the result of the expression */ + /* if dvomath returns NULL, the expression was not a math operation: treat as a string */ val = dvomath (1, &V1, &size, 0); if (val == NULL) { while (OHANA_WHITESPACE (*V1)) V1++; val = strcreate (V1); + init_error (); } // save the result @@ -245,5 +247,5 @@ vec[0].elements.Flt[Nx] = atof (val); } else { - vec[0].elements.Int[Nx] = atol (val); + vec[0].elements.Int[Nx] = atoll (val); } } Index: /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/stack_math.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/stack_math.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/lib.shell/stack_math.c (revision 40477) @@ -599,5 +599,5 @@ } -// the vector is applied to each column (currently only valid for 2D matrix) +// the vector is applied to each ROW (currently only valid for 2D matrix), e.g.: M[20,10] * X[20] -> M'[20,10] where M'[2,1] = M[2,1] * X[2] int MV_binary (StackVar *OUT, StackVar *V1, StackVar *V2, char *op) { @@ -607,6 +607,7 @@ Nx = V1[0].buffer[0].matrix.Naxis[0]; Ny = V1[0].buffer[0].matrix.Naxis[1]; - if (Ny != V2[0].vector[0].Nelements) { - push_error ("dimension mismatch"); + if (Nx != V2[0].vector[0].Nelements) { + snprintf (line, 512, "error: matrix OP vector matrix dimensions do not match: (%d x %d) OP %d", Nx, Ny, V2[0].vector[0].Nelements); + push_error (line); return (FALSE); } @@ -628,7 +629,7 @@ # define MV_FUNC(OP) { \ if (V2->vector->type == OPIHI_FLT) { \ - opihi_flt *M2 = V2[0].vector[0].elements.Flt; \ - for (i = 0; i < Ny; i++, M2++) { \ - for (j = 0; j < Nx; j++, out++, M1++) { \ + for (i = 0; i < Ny; i++) { \ + opihi_flt *M2 = V2[0].vector[0].elements.Flt; \ + for (j = 0; j < Nx; j++, out++, M1++, M2++) { \ *out = OP; \ } \ @@ -637,7 +638,7 @@ } \ if (V2->vector->type != OPIHI_FLT) { \ - opihi_int *M2 = V2[0].vector[0].elements.Int; \ - for (i = 0; i < Ny; i++, M2++) { \ - for (j = 0; j < Nx; j++, out++, M1++) { \ + for (i = 0; i < Ny; i++) { \ + opihi_int *M2 = V2[0].vector[0].elements.Int; \ + for (j = 0; j < Nx; j++, out++, M1++, M2++) { \ *out = OP; \ } \ @@ -695,5 +696,6 @@ } -// the vector is applied to each row (currently only valid for 2D matrix) +// the vector is applied to each COLUMN (currently only valid for 2D matrix) +// e.g.: X[10] * M[20,10] -> M'[20,10] where M'[2,1] = X[1] * M[2,1] int VM_binary (StackVar *OUT, StackVar *V1, StackVar *V2, char *op) { @@ -703,5 +705,7 @@ Nx = V2[0].buffer[0].matrix.Naxis[0]; Ny = V2[0].buffer[0].matrix.Naxis[1]; - if (Nx != V1[0].vector[0].Nelements) { + if (Ny != V1[0].vector[0].Nelements) { + snprintf (line, 512, "error: vector OP matrix dimensions do not match: %d OP (%d x %d)", V1[0].vector[0].Nelements, Nx, Ny); + push_error (line); return (FALSE); } @@ -722,7 +726,7 @@ # define VM_FUNC(OP) { \ if (V1->vector->type == OPIHI_FLT) { \ - for (i = 0; i < Ny; i++) { \ - opihi_flt *M1 = V1[0].vector[0].elements.Flt; \ - for (j = 0; j < Nx; j++, out++, M1++, M2++) { \ + opihi_flt *M1 = V1[0].vector[0].elements.Flt; \ + for (i = 0; i < Ny; i++, M1++) { \ + for (j = 0; j < Nx; j++, out++, M2++) { \ *out = OP; \ } \ @@ -731,7 +735,7 @@ } \ if (V1->vector->type != OPIHI_FLT) { \ - for (i = 0; i < Ny; i++) { \ - opihi_int *M1 = V1[0].vector[0].elements.Int; \ - for (j = 0; j < Nx; j++, out++, M1++, M2++) { \ + opihi_int *M1 = V1[0].vector[0].elements.Int; \ + for (i = 0; i < Ny; i++, M1++) { \ + for (j = 0; j < Nx; j++, out++, M2++) { \ *out = OP; \ } \ Index: /branches/czw_branch/20170908/Ohana/src/opihi/mana/findrowpeaks.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/mana/findrowpeaks.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/mana/findrowpeaks.c (revision 40477) @@ -23,4 +23,5 @@ for (i = 1; i < Nrow - 1; i++) { + if (!isfinite(row[i])) continue; // ignore NAN values if (row[i] < threshold) continue; if (row[i] < row[i-1]) continue; Index: /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/CheckController.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/CheckController.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/CheckController.c (revision 40477) @@ -7,5 +7,6 @@ char *p, *q; - int i, Njobs, status, JobID; + int i, Njobs, status; + IDtype JobID; Job *job; IOBuffer buffer; Index: /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/JobIDOps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/JobIDOps.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/JobIDOps.c (revision 40477) @@ -1,44 +1,26 @@ # include "pantasks.h" -# define MAX_N_JOBS 10000 -static char *JobIDList; -static int JobIDPtr; +# define MAX_N_JOBS 0x80000000 + +static IDtype JobIDMax = 0; void InitJobIDs () { - - JobIDPtr = 0; - ALLOCATE (JobIDList, char, MAX_N_JOBS); - bzero (JobIDList, MAX_N_JOBS*sizeof(char)); + JobIDMax = 0; } -void FreeJobIDs () { - free (JobIDList); +void FreeJobIDs () { } + +/* return next unique ID, recycle every MAX_N_JOBS */ +IDtype NextJobID () { + + JobIDMax ++; + if (JobIDMax >= MAX_N_JOBS) { + gprint (GP_ERR, "ERROR: too many jobs spawned: %d, aborting\n", JobIDMax); + abort(); + } + + return (JobIDMax); } -/* return next unique ID, recycle every MAX_N_JOBS */ -int NextJobID () { - - int Ntry; - - JobIDPtr ++; - if (JobIDPtr >= MAX_N_JOBS) JobIDPtr = 0; - - Ntry = 0; - while (JobIDList[JobIDPtr]) { - Ntry ++; - JobIDPtr ++; - if (JobIDPtr >= MAX_N_JOBS) JobIDPtr = 0; - if (Ntry == MAX_N_JOBS) return (-1); - } - JobIDList[JobIDPtr] = TRUE; - return (JobIDPtr); -} - -int FreeJobID (int JobID) { - - if (JobID < 0) return (FALSE); - if (JobID >= MAX_N_JOBS) return (FALSE); - - JobIDList[JobID] = FALSE; - return (TRUE); -} +/* I am going to rework the jobID so that it just grows. no need to ever free them + as an int, that allows 2^31 jobs (2e9 jobs) */ Index: /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/JobOps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/JobOps.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/JobOps.c (revision 40477) @@ -41,5 +41,5 @@ /* return job with given ID */ -Job *FindJob (int JobID) { +Job *FindJob (IDtype JobID) { int i; @@ -55,5 +55,5 @@ /* return job with given controller Job ID */ -Job *FindControllerJob (int JobID) { +Job *FindControllerJob (IDtype JobID) { int i; @@ -82,5 +82,5 @@ gprint (GP_LOG, " Jobs in Pantasks Queue\n"); for (i = 0; i < Njobs; i++) { - gprint (GP_LOG, " %4d %6d: %-25s %10s %20s\n", Njobs, jobs[i][0].JobID, jobs[i][0].task[0].name, JobStateToString(jobs[i][0].state), jobs[i][0].argv[0]); + gprint (GP_LOG, " %4d %8d: %-25s %10s %20s\n", Njobs, jobs[i][0].JobID, jobs[i][0].task[0].name, JobStateToString(jobs[i][0].state), jobs[i][0].argv[0]); } @@ -97,4 +97,6 @@ job[0].JobID = NextJobID (); + + // this should not happen; abort? if (job[0].JobID < 0) { free (job); @@ -165,6 +167,4 @@ if (job == NULL) return; - FreeJobID (job[0].JobID); - for (i = 0; i < job[0].argc; i++) { free (job[0].argv[i]); Index: /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/delete.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/delete.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/delete.c (revision 40477) @@ -4,5 +4,5 @@ Job *job; - int JobID; + IDtype JobID; if (argc != 3) goto usage; Index: /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/kill.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/kill.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/kill.c (revision 40477) @@ -4,5 +4,5 @@ Job *job; - int JobID; + IDtype JobID; if (argc < 2) { @@ -10,5 +10,5 @@ return (FALSE); } - JobID = atoi (argv[1]); + JobID = atoll (argv[1]); JobTaskLock(); Index: /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/test/nice_remote.sh =================================================================== --- /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/test/nice_remote.sh (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/opihi/pantasks/test/nice_remote.sh (revision 40477) @@ -10,5 +10,5 @@ periods -timeout 20 active true - npending 5 + npending 50 stdout tmp.txt @@ -44,5 +44,5 @@ periods -timeout 20 active true - npending 5 + npending 50 stdout tmp.txt Index: /branches/czw_branch/20170908/Ohana/src/photdbc/src/get_mags.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/photdbc/src/get_mags.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/photdbc/src/get_mags.c (revision 40477) @@ -20,7 +20,7 @@ Nlist = MAX (Nlist, catalog[0].average[i].Nm); for (j = 0; j < Nsecfilt; j++) { - catalog[0].secfilt[i*Nsecfilt+j].M = NAN; - catalog[0].secfilt[i*Nsecfilt+j].dM = NAN; - catalog[0].secfilt[i*Nsecfilt+j].Mchisq = NAN; + catalog[0].secfilt[i*Nsecfilt+j].MpsfChp = NAN; + catalog[0].secfilt[i*Nsecfilt+j].dMpsfChp = NAN; + catalog[0].secfilt[i*Nsecfilt+j].Mchisq = NAN; } } @@ -52,7 +52,7 @@ if (N < 1) continue; - Mval = (Nsec == -1) ? &catalog[0].average[i].M : &catalog[0].secfilt[i*Nsecfilt+Nsec].M; + Mval = (Nsec == -1) ? &catalog[0].average[i].M : &catalog[0].secfilt[i*Nsecfilt+Nsec].MpsfChp; *Mval = stats.mean; - Mval = (Nsec == -1) ? &catalog[0].average[i].dM : &catalog[0].secfilt[i*Nsecfilt+Nsec].dM; + Mval = (Nsec == -1) ? &catalog[0].average[i].dM : &catalog[0].secfilt[i*Nsecfilt+Nsec].dMpsfChp; *Mval = stats.sigma; Mval = (Nsec == -1) ? &catalog[0].average[i].Mchisq : &catalog[0].secfilt[i*Nsecfilt+Nsec].Mchisq; Index: /branches/czw_branch/20170908/Ohana/src/photdbc/src/join_stars.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/photdbc/src/join_stars.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/photdbc/src/join_stars.c (revision 40477) @@ -87,5 +87,5 @@ // require base star to meet certain conditions: - if (isnan(secfilt[Ni*Nsecfilt + baseNsec].M)) { + if (isnan(secfilt[Ni*Nsecfilt + baseNsec].MpsfChp)) { i++; continue; Index: /branches/czw_branch/20170908/Ohana/src/photdbc/src/make_subcatalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/photdbc/src/make_subcatalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/photdbc/src/make_subcatalog.c (revision 40477) @@ -1,3 +1,9 @@ # include "photdbc.h" + +# define SKIP_MEASURE FALSE +# define SKIP_LENSING FALSE +# define SKIP_LENSOBJ FALSE +# define SKIP_STARPAR FALSE +# define SKIP_GALPHOT FALSE // copy a catalog to a new subcatalog, applying some filters @@ -63,5 +69,5 @@ keep = FALSE; for (j = 0; !keep && (j < Nsecfilt); j++) { - if (catalog[0].secfilt[Nsecfilt*i+j].dM < AVE_SIGMA_LIM) { + if (catalog[0].secfilt[Nsecfilt*i+j].dMpsfChp < AVE_SIGMA_LIM) { keep = TRUE; } @@ -98,5 +104,5 @@ minSigma = 32; Nm = 0; - for (j = 0; j < catalog[0].average[i].Nmeasure; j++) { + for (j = 0; !SKIP_MEASURE && (j < catalog[0].average[i].Nmeasure); j++) { offset = catalog[0].average[i].measureOffset + j; @@ -164,4 +170,5 @@ } } + // End of average[i] loop // exclude faint objects @@ -189,5 +196,5 @@ Nm = 0; subcatalog[0].average[Naverage].lensingOffset = Nlensing; - for (j = 0; j < catalog[0].average[i].Nlensing; j++) { + for (j = 0; !SKIP_LENSING && (j < catalog[0].average[i].Nlensing); j++) { offset = catalog[0].average[i].lensingOffset + j; @@ -208,5 +215,5 @@ Nm = 0; subcatalog[0].average[Naverage].lensobjOffset = Nlensobj; - for (j = 0; j < catalog[0].average[i].Nlensobj; j++) { + for (j = 0; !SKIP_LENSOBJ && (j < catalog[0].average[i].Nlensobj); j++) { offset = catalog[0].average[i].lensobjOffset + j; @@ -226,5 +233,5 @@ Nm = 0; subcatalog[0].average[Naverage].starparOffset = Nstarpar; - for (j = 0; j < catalog[0].average[i].Nstarpar; j++) { + for (j = 0; !SKIP_STARPAR && (j < catalog[0].average[i].Nstarpar); j++) { offset = catalog[0].average[i].starparOffset + j; @@ -245,5 +252,5 @@ Nm = 0; subcatalog[0].average[Naverage].galphotOffset = Ngalphot; - for (j = 0; j < catalog[0].average[i].Ngalphot; j++) { + for (j = 0; !SKIP_GALPHOT && (j < catalog[0].average[i].Ngalphot); j++) { offset = catalog[0].average[i].galphotOffset + j; Index: /branches/czw_branch/20170908/Ohana/src/relastro/Makefile =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/Makefile (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/Makefile (revision 40477) @@ -1,3 +1,3 @@ -default: relastro relastro_client fitobj fitobj2 fitobj3 fitpm +default: relastro relastro_client fitobj fitobj2 fitobj3 fitexample help: @echo "make options: relastro (default)" @@ -109,5 +109,6 @@ $(SRC)/relastroVisual.$(ARCH).o -#$(SRC)/write_coords.$(ARCH).o \ +#$(SRC)/write_coords.$(ARCH).o +#$(SRC)/UpdateStacksWithFit.$(ARCH).o $(RELASTRO): $(INC)/relastro.h @@ -187,20 +188,20 @@ $(BIN)/testparallax.$(ARCH): $(TESTPAR) -fitpm: $(BIN)/fitpm.$(ARCH) - -FITPM = \ -$(SRC)/fitpm.$(ARCH).o \ -$(SRC)/ParFactor.$(ARCH).o \ -$(SRC)/FitAstromOps.$(ARCH).o \ -$(SRC)/FitPosPMfixed.$(ARCH).o \ -$(SRC)/FitPM.$(ARCH).o \ -$(SRC)/FitPMandPar.$(ARCH).o \ -$(SRC)/FitPosStack.$(ARCH).o \ -$(SRC)/BootstrapOps.$(ARCH).o \ -$(SRC)/mkpolyterm.$(ARCH).o \ -$(SRC)/fitpoly.$(ARCH).o - -$(FITPM): $(INC)/relastro.h -$(BIN)/fitpm.$(ARCH): $(FITPM) +fitexample: $(BIN)/fitexample.$(ARCH) + +FITEXAMPLE = \ +$(SRC)/fitexample.$(ARCH).o \ +$(SRC)/ParFactor.$(ARCH).o \ +$(SRC)/FitAstromOps.$(ARCH).o \ +$(SRC)/FitPosPMfixed.$(ARCH).o \ +$(SRC)/FitPM.$(ARCH).o \ +$(SRC)/FitPMandPar.$(ARCH).o \ +$(SRC)/FitPosStack.$(ARCH).o \ +$(SRC)/BootstrapOps.$(ARCH).o \ +$(SRC)/mkpolyterm.$(ARCH).o \ +$(SRC)/fitpoly.$(ARCH).o + +$(FITEXAMPLE): $(INC)/relastro.h +$(BIN)/fitexample.$(ARCH): $(FITEXAMPLE) fitobj: $(BIN)/fitobj.$(ARCH) Index: /branches/czw_branch/20170908/Ohana/src/relastro/include/relastro.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/include/relastro.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/include/relastro.h (revision 40477) @@ -33,5 +33,5 @@ typedef enum {OP_NONE, OP_IMAGES, OP_HIGH_SPEED, OP_MERGE_SOURCE, OP_UPDATE_OBJECTS, OP_UPDATE_OFFSETS, OP_LOAD_OBJECTS, OP_HPM, OP_PARALLEL_REGIONS, OP_PARALLEL_IMAGES, OP_REPAIR_STACKS, OP_REPAIR_WARPS, OP_REPAIR_OBJECT_ID} RelastroOp; -typedef enum {TARGET_NONE, TARGET_SIMPLE, TARGET_CHIPS, SET_CHIPS, TARGET_MOSAICS} FitTarget; +typedef enum {TARGET_NONE, TARGET_SIMPLE, TARGET_CHIPS, SET_CHIPS, SET_STACKS, TARGET_MOSAICS} FitTarget; typedef enum { @@ -215,7 +215,8 @@ unsigned int stop; off_t myImage; - float Mcal; + float McalPSF; + float McalAPER; float dMcal; - short Xm; + float McalChiSq; float secz; char flags; @@ -366,4 +367,6 @@ int USE_ICRF_POLE; +int FIT_STACKS; +int IMSTATS_ONLY; int REPAIR_STACKS; int USE_IMAGE_COORDS_FOR_REPAIR; @@ -377,4 +380,5 @@ int SKIP_CFH; +int UPDATE_ALL_MEASURE; int UPDATE_PS1_STACK_MEASURE; int UPDATE_PS1_CHIP_MEASURE; @@ -621,4 +625,5 @@ int UpdateChips (Catalog *catalog, int Ncatalog, int Nloop); int UpdateStacks (Catalog *catalog, int Ncatalog); +int UpdateStacksWithFit (Catalog *catalog, int Ncatalog); int UpdateMosaic (Catalog *catalog, int Ncatalog); int UpdateMeasures (Catalog *catalog, int Ncatalog); Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/BrightCatalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/BrightCatalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/BrightCatalog.c (revision 40477) @@ -68,5 +68,6 @@ GET_COLUMN(D, "DEC", double); GET_COLUMN(M, "MAG_SYS", float); - GET_COLUMN(Mcal, "MAG_CAL", float); + GET_COLUMN(McalPSF, "MCAL_PSF", float); + GET_COLUMN(McalAPER, "MCAL_APER", float); GET_COLUMN(dM, "MAG_ERR", float); GET_COLUMN(airmass, "AIRMASS", float); @@ -94,5 +95,6 @@ measure[i].D = D[i]; measure[i].M = M[i]; - measure[i].Mcal = Mcal[i]; + measure[i].McalPSF = McalPSF[i]; + measure[i].McalAPER = McalAPER[i]; measure[i].dM = dM[i]; measure[i].airmass = airmass[i]; @@ -120,5 +122,6 @@ free (D ); free (M ); - free (Mcal ); + free (McalPSF ); + free (McalAPER); free (dM ); free (airmass ); @@ -285,12 +288,12 @@ ALLOCATE (secfilt, SecFilt, Nrow); for (i = 0; i < Nrow; i++) { - secfilt[i].M = M[i]; - secfilt[i].dM = dM[i]; - secfilt[i].Mchisq = Mchisq[i]; - secfilt[i].flags = flags[i]; - secfilt[i].Ncode = Ncode[i]; - secfilt[i].Nused = Nused[i]; - secfilt[i].Mmin = Mmin[i]; - secfilt[i].Mmax = Mmax[i]; + secfilt[i].MpsfChp = M[i]; + secfilt[i].dMpsfChp = dM[i]; + secfilt[i].Mchisq = Mchisq[i]; + secfilt[i].flags = flags[i]; + secfilt[i].Ncode = Ncode[i]; + secfilt[i].Nused = Nused[i]; + secfilt[i].Mmin = Mmin[i]; + secfilt[i].Mmax = Mmax[i]; } // fprintf (stderr, "loaded data for %lld secfilt\n", (long long) Nrow); @@ -361,5 +364,6 @@ gfits_define_bintable_column (&theader, "D", "DEC", "dec", "degrees", 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "MAG_SYS", "magnitude (sys)", NULL, 1.0, 0.0); - gfits_define_bintable_column (&theader, "E", "MAG_CAL", "magnitude (cal)", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_PSF", "magnitude (cal)", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_APER","magnitude (cal)", NULL, 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "MAG_ERR", "magnitude (err)", NULL, 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "AIRMASS", "airmass", NULL, 1.0, 0.0); @@ -385,5 +389,6 @@ double *D ; ALLOCATE (D , double, catalog->Nmeasure); float *M ; ALLOCATE (M , float, catalog->Nmeasure); - float *Mcal ; ALLOCATE (Mcal , float, catalog->Nmeasure); + float *McalPSF ; ALLOCATE (McalPSF , float, catalog->Nmeasure); + float *McalAPER ; ALLOCATE (McalAPER , float, catalog->Nmeasure); float *dM ; ALLOCATE (dM , float, catalog->Nmeasure); float *airmass ; ALLOCATE (airmass , float, catalog->Nmeasure); @@ -408,5 +413,6 @@ D[i] = measure[i].D ; M[i] = measure[i].M ; - Mcal[i] = measure[i].Mcal ; + McalPSF[i] = measure[i].McalPSF ; + McalAPER[i] = measure[i].McalAPER ; dM[i] = measure[i].dM ; airmass[i] = measure[i].airmass ; @@ -434,5 +440,6 @@ gfits_set_bintable_column (&theader, &ftable, "DEC", D, catalog->Nmeasure); gfits_set_bintable_column (&theader, &ftable, "MAG_SYS", M, catalog->Nmeasure); - gfits_set_bintable_column (&theader, &ftable, "MAG_CAL", Mcal, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MCAL_PSF", McalPSF, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MCAL_APER", McalAPER, catalog->Nmeasure); gfits_set_bintable_column (&theader, &ftable, "MAG_ERR", dM, catalog->Nmeasure); gfits_set_bintable_column (&theader, &ftable, "AIRMASS", airmass, catalog->Nmeasure); @@ -454,5 +461,6 @@ free (D ); free (M ); - free (Mcal ); + free (McalPSF ); + free (McalAPER); free (dM ); free (airmass ); @@ -694,12 +702,12 @@ SecFilt *secfilt = catalog->secfilt; for (i = 0; i < Nsec; i++) { - M [i] = secfilt[i].M ; - dM [i] = secfilt[i].dM ; - Mchisq[i] = secfilt[i].Mchisq ; - flags [i] = secfilt[i].flags ; - Ncode [i] = secfilt[i].Ncode ; - Nused [i] = secfilt[i].Nused ; - Mmin [i] = secfilt[i].Mmin ; - Mmax [i] = secfilt[i].Mmax ; + M [i] = secfilt[i].MpsfChp ; + dM [i] = secfilt[i].dMpsfChp; + Mchisq[i] = secfilt[i].Mchisq ; + flags [i] = secfilt[i].flags ; + Ncode [i] = secfilt[i].Ncode ; + Nused [i] = secfilt[i].Nused ; + Mmin [i] = secfilt[i].Mmin ; + Mmax [i] = secfilt[i].Mmax ; } Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/ImageOps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/ImageOps.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/ImageOps.c (revision 40477) @@ -431,5 +431,5 @@ /**** dMcal vs airmass ****/ for (i = 0; i < Nimage; i++) { - Mlist[i] = image[i].Mcal; + Mlist[i] = image[i].McalPSF; dlist[i] = image[i].dMcal; xlist[i] = image[i].secz; @@ -819,8 +819,8 @@ if ((DCR_BLUE_NSEC_POS >= 0) && (DCR_BLUE_NSEC_NEG >= -1)) { - ref[i].ColorBlue = catalog[c].secfilt[n*Nsecfilt + DCR_BLUE_NSEC_POS].M - catalog[c].secfilt[n*Nsecfilt + DCR_BLUE_NSEC_NEG].M; + ref[i].ColorBlue = catalog[c].secfilt[n*Nsecfilt + DCR_BLUE_NSEC_POS].MpsfChp - catalog[c].secfilt[n*Nsecfilt + DCR_BLUE_NSEC_NEG].MpsfChp; } if ((DCR_RED_NSEC_POS >= 0) && (DCR_RED_NSEC_NEG >= -1)) { - ref[i].ColorRed = catalog[c].secfilt[n*Nsecfilt + DCR_RED_NSEC_POS].M - catalog[c].secfilt[n*Nsecfilt + DCR_RED_NSEC_NEG].M; + ref[i].ColorRed = catalog[c].secfilt[n*Nsecfilt + DCR_RED_NSEC_POS].MpsfChp - catalog[c].secfilt[n*Nsecfilt + DCR_RED_NSEC_NEG].MpsfChp; } Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/MosaicOps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/MosaicOps.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/MosaicOps.c (revision 40477) @@ -97,13 +97,14 @@ /* a new mosaic, define ranges */ - mosaic[Nmosaic].start = start; - mosaic[Nmosaic].stop = stop; - mosaic[Nmosaic].Mcal = 0.0; - mosaic[Nmosaic].dMcal = 0.0; - mosaic[Nmosaic].Xm = 0.0; - mosaic[Nmosaic].flags = image[i].flags; - mosaic[Nmosaic].secz = image[i].secz; - mosaic[Nmosaic].coords = image[i].coords; - mosaic[Nmosaic].myImage = i; + mosaic[Nmosaic].start = start; + mosaic[Nmosaic].stop = stop; + mosaic[Nmosaic].McalPSF = 0.0; + mosaic[Nmosaic].McalAPER = 0.0; + mosaic[Nmosaic].dMcal = 0.0; + mosaic[Nmosaic].McalChiSq = 0.0; + mosaic[Nmosaic].flags = image[i].flags; + mosaic[Nmosaic].secz = image[i].secz; + mosaic[Nmosaic].coords = image[i].coords; + mosaic[Nmosaic].myImage = i; // init the mosaic_own_images array data Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateChips.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateChips.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateChips.c (revision 40477) @@ -88,4 +88,5 @@ raw = getImageRaw (catalog, Ncatalog, i, &Nraw, MODE_MOSAIC); if (!raw) { + // fprintf (stderr, "skip 1 %s\n", image[i].name); Nskip ++; mode[i] = 0; @@ -93,4 +94,5 @@ } if (Nraw <= IMFIT_TOO_FEW) { + // fprintf (stderr, "skip 2 %s\n", image[i].name); Nskip ++; mode[i] = 0; @@ -102,4 +104,5 @@ ref = getImageRef (catalog, Ncatalog, i, &Nref, MODE_MOSAIC); if (!ref) { + // fprintf (stderr, "skip 3 %s\n", image[i].name); Nskip ++; mode[i] = 0; @@ -116,4 +119,19 @@ // note that Nraw & Nref must be equal: if not, we made a programming error in one of these two functions. assert (Nraw == Nref); + + if (IMSTATS_ONLY) { + int Nstat; + float dLsig, dMsig, dRsig; + GetScatterRawRef(&dLsig, &dMsig, &dRsig, &Nstat, raw, ref, Nraw, IMFIT_SYS_SIGMA_LIM); + + // XXX: I need to convert dLsig, dMsig from degrees to pixels + dLsig *= 3600.0; + dMsig *= 3600.0; + + image[i].dXpixSys = dLsig; + image[i].dYpixSys = dMsig; + image[i].nFitAstrom = Nstat; + continue; + } // save these in case of failure Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateMeasures.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateMeasures.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateMeasures.c (revision 40477) @@ -47,5 +47,25 @@ } - off_t im = getImageByID (measureT->imageID); + Average *average = &catalog[i].average[measureB->averef]; + int FOUND_TEST = FALSE; + FOUND_TEST |= (average[0].objID == OBJ_ID_SRC) && (average[0].catID == CAT_ID_SRC); + FOUND_TEST |= (average[0].objID == OBJ_ID_DST) && (average[0].catID == CAT_ID_DST); + if (FOUND_TEST) { + fprintf (stderr, "found test object\n"); + } + + // we need this below but cannot declare it after the label update_this_measure: + off_t im; + + // we have a few options to include/exclude specific types of measurements + if (UPDATE_ALL_MEASURE) goto update_this_measure; + if (isGPC1chip (measureT->photcode) && UPDATE_PS1_CHIP_MEASURE) goto update_this_measure; + if (isGPC1stack (measureT->photcode) && UPDATE_PS1_STACK_MEASURE) goto update_this_measure; + if (isHSCchip (measureT->photcode) && UPDATE_HSC_MEASURE) goto update_this_measure; + if (isCFHchip (measureT->photcode) && UPDATE_CFH_MEASURE) goto update_this_measure; + continue; // skip all others + + update_this_measure: + im = getImageByID (measureT->imageID); if (im < 0) continue; // detections without imageIDs are not associated with a known image (eg, refs) @@ -114,39 +134,10 @@ } - // only modify the chip coordinates - if (UPDATE_PS1_STACK_MEASURE && isGPC1stack (measureT->photcode)) { - measureT->R = R; - measureT->D = D; - if (measureB) { - measureB->R = R; - measureB->D = D; - } - } - // only modify the chip coordinates - if (UPDATE_PS1_CHIP_MEASURE && isGPC1chip (measureT->photcode)) { - measureT->R = R; - measureT->D = D; - if (measureB) { - measureB->R = R; - measureB->D = D; - } - } - // only modify the chip coordinates - if (UPDATE_HSC_MEASURE && isGPC1chip (measureT->photcode)) { - measureT->R = R; - measureT->D = D; - if (measureB) { - measureB->R = R; - measureB->D = D; - } - } - // only modify the chip coordinates - if (UPDATE_CFH_MEASURE && isGPC1chip (measureT->photcode)) { - measureT->R = R; - measureT->D = D; - if (measureB) { - measureB->R = R; - measureB->D = D; - } + // modify the measure coordinates + measureT->R = R; + measureT->D = D; + if (measureB) { + measureB->R = R; + measureB->D = D; } } Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateObjectOffsets.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 40477) @@ -208,13 +208,14 @@ if (CHECK_MEASURE_TO_IMAGE) strextend (&command, "-check-measures"); + if (UPDATE_ALL_MEASURE) strextend (&command, "-update-all-cameras"); if (UPDATE_PS1_STACK_MEASURE) strextend (&command, "-update-ps1-stack"); - if (UPDATE_PS1_CHIP_MEASURE) strextend (&command, "-update-ps1-chip"); - if (UPDATE_HSC_MEASURE) strextend (&command, "-update-hsc"); - if (UPDATE_CFH_MEASURE) strextend (&command, "-update-cfh"); - - if (UPDATE) strextend (&command, "-update"); + if (UPDATE_PS1_CHIP_MEASURE) strextend (&command, "-update-ps1-chip"); + if (UPDATE_HSC_MEASURE) strextend (&command, "-update-hsc"); + if (UPDATE_CFH_MEASURE) strextend (&command, "-update-cfh"); + + if (UPDATE) strextend (&command, "-update"); if (RESET_BAD_IMAGES) strextend (&command, "-reset-bad-images"); - if (USE_BASIC_CHECK) strextend (&command, "-basic-image-search"); - if (USE_ALL_IMAGES) strextend (&command, "-use-all-images"); + if (USE_BASIC_CHECK) strextend (&command, "-basic-image-search"); + if (USE_ALL_IMAGES) strextend (&command, "-use-all-images"); if (MinBadQF > 0.0) strextend (&command, "-min-bad-psfqf %f", MinBadQF); @@ -347,8 +348,9 @@ if (CHECK_MEASURE_TO_IMAGE) strextend (&command, "-check-measures"); + if (UPDATE_ALL_MEASURE) strextend (&command, "-update-all-cameras"); if (UPDATE_PS1_STACK_MEASURE) strextend (&command, "-update-ps1-stack"); - if (UPDATE_PS1_CHIP_MEASURE) strextend (&command, "-update-ps1-chip"); - if (UPDATE_HSC_MEASURE) strextend (&command, "-update-hsc"); - if (UPDATE_CFH_MEASURE) strextend (&command, "-update-cfh"); + if (UPDATE_PS1_CHIP_MEASURE) strextend (&command, "-update-ps1-chip"); + if (UPDATE_HSC_MEASURE) strextend (&command, "-update-hsc"); + if (UPDATE_CFH_MEASURE) strextend (&command, "-update-cfh"); if (UPDATE) strextend (&command, "-update"); Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateObjects.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateObjects.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateObjects.c (revision 40477) @@ -127,6 +127,12 @@ int k; + // we need to chose to do something here... + int valid = UPDATE_ALL_MEASURE; + valid |= UPDATE_PS1_CHIP_MEASURE; + valid |= UPDATE_HSC_MEASURE; + valid |= UPDATE_CFH_MEASURE; + if (!valid) return FALSE; + /* calculate the average value of R,D for a single star */ - FitAstromResult fitPos, fitPM, fitPar; FitAstromResultInit (&fitPos); @@ -583,4 +589,7 @@ int status; + // we need to chose to do something here... + if (!UPDATE_PS1_STACK_MEASURE) return FALSE; + // set the default values average[0].Rstk = NAN; // RA in degrees @@ -794,4 +803,5 @@ Npoints++; + // NOTE: if 'isStack' is true, we will never see these photcode values hasGAIA = (measure[k].photcode == 1030); has2MASS = ((measure[k].photcode >= 2011) && (measure[k].photcode <= 2013)); @@ -808,7 +818,9 @@ } - // XXX flag measurements from stars with 2MASS - for (k = 0; k < average[0].Nmeasure; k++) { - // reset the bit to note that a detection was used (or not) + // flag measurements from stars with 2MASS, GAIA, TYCHO + // NOTE: if 'isStack' is TRUE, these will not be correctly set. + // ONLY do this loop if not(isStack) + for (k = 0; !isStack && (k < average[0].Nmeasure); k++) { + // reset the bit to note that an object does or does not have the value if (has2MASS) { measure[k].dbFlags |= ID_MEAS_OBJECT_HAS_2MASS; Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateStacks.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateStacks.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateStacks.c (revision 40477) @@ -11,4 +11,6 @@ off_t Nskip = 0; off_t Nmeas = 0; + off_t NnewFit = 0; + off_t NoldFit = 0; // measure the scatter for each stack @@ -43,15 +45,54 @@ assert (Nraw == Nref); - int Nstat; - float dLsig, dMsig, dRsig; - GetScatterRawRef(&dLsig, &dMsig, &dRsig, &Nstat, raw, ref, Nraw, IMFIT_SYS_SIGMA_LIM); + // the natural default for stacks is to NOT re-fit + if (FIT_STACKS == FALSE) { + int Nstat; + float dLsig, dMsig, dRsig; + GetScatterRawRef(&dLsig, &dMsig, &dRsig, &Nstat, raw, ref, Nraw, IMFIT_SYS_SIGMA_LIM); - // XXX: I need to convert dLsig, dMsig from degrees to pixels - dLsig *= 3600.0; - dMsig *= 3600.0; + // XXX: I need to convert dLsig, dMsig from degrees to pixels + dLsig *= 3600.0; + dMsig *= 3600.0; - image[i].dXpixSys = dLsig; - image[i].dYpixSys = dMsig; - image[i].nFitAstrom = Nstat; + image[i].dXpixSys = dLsig; + image[i].dYpixSys = dMsig; + image[i].nFitAstrom = Nstat; + continue; + } + + // save these in case of failure + Coords oldCoords; + SaveCoords (&oldCoords, &image[i].coords); + + float dXpixSys = image[i].dXpixSys; + float dYpixSys = image[i].dYpixSys; + int nFitAstr = image[i].nFitAstrom; + + // FitChip does iterative, clipped fitting + if (!FitChip (raw, ref, Nraw, &image[i])) { + if (VERBOSE) fprintf (stderr, "reject fit for image %s ("OFF_T_FMT") : Nstars: "OFF_T_FMT", Nused %d of %d\n", image[i].name, i, Nraw, image[i].nFitAstrom, image[i].nstar); + + // restore status quo ante (replace truMap with tmpMap) + RestoreCoords (&image[i].coords, &oldCoords, &image[i]); + image[i].dXpixSys = dXpixSys; + image[i].dYpixSys = dYpixSys; + image[i].nFitAstrom = nFitAstr; + + NoldFit ++; + free (raw); + free (ref); + continue; + } + + AstromOffsetMapFree (oldCoords.offsetMap); + + // Apply the modified coords back to the measure.R,D. Note that raw.R,D, ref.L,M, etc + // are all automatically updated in this block because they are re-generated from + // image.coords on each pass. + setImageRaw (catalog, Ncatalog, i, raw, Nraw, MODE_MOSAIC); + + NnewFit ++; + free (raw); + free (ref); } Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateStacksWithFit.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateStacksWithFit.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/UpdateStacksWithFit.c (revision 40477) @@ -0,0 +1,86 @@ +# include "relastro.h" + +// This stack analysis allows us to fit the chip model to each stack +int UpdateStacksWithFit (Catalog *catalog, int Ncatalog) { + + if (SKIP_PS1_STACK) return TRUE; + + off_t Nimage; + Image *image = getimages (&Nimage, NULL); + + off_t Nskip = 0; + off_t NnewFit = 0; + off_t NoldFit = 0; + + // measure the scatter for each stack + for (off_t i = 0; i < Nimage; i++) { + + /* skip all except stack images */ + if (!isGPC1stack(image[i].photcode)) continue; + + /* convert measure coordinates to raw entries */ + off_t Nraw; + StarData *raw = getImageRaw (catalog, Ncatalog, i, &Nraw, MODE_SIMPLE); + if (!raw) { + Nskip ++; + continue; + } + if (Nraw <= IMFIT_TOO_FEW) { + Nskip ++; + free (raw); + continue; + } + + /* convert average coordinates to ref entries */ + off_t Nref; + StarData *ref = getImageRef (catalog, Ncatalog, i, &Nref, MODE_SIMPLE); + if (!ref) { + Nskip ++; + free (raw); + continue; + } + + // note that Nraw & Nref must be equal: if not, we made a programming error in one of these two functions. + assert (Nraw == Nref); + + // save these in case of failure + Coords oldCoords; + SaveCoords (&oldCoords, &image[i].coords); + + float dXpixSys = image[i].dXpixSys; + float dYpixSys = image[i].dYpixSys; + int nFitAstr = image[i].nFitAstrom; + + // FitChip does iterative, clipped fitting + if (!FitChip (raw, ref, Nraw, &image[i])) { + if (VERBOSE) fprintf (stderr, "reject fit for image %s ("OFF_T_FMT") : Nstars: "OFF_T_FMT", Nused %d of %d\n", image[i].name, i, Nraw, image[i].nFitAstrom, image[i].nstar); + + // restore status quo ante (replace truMap with tmpMap) + RestoreCoords (&image[i].coords, &oldCoords, &image[i]); + image[i].dXpixSys = dXpixSys; + image[i].dYpixSys = dYpixSys; + image[i].nFitAstrom = nFitAstr; + + NoldFit ++; + free (raw); + free (ref); + continue; + } + + AstromOffsetMapFree (oldCoords.offsetMap); + + // Apply the modified coords back to the measure.R,D. Note that raw.R,D, ref.L,M, etc + // are all automatically updated in this block because they are re-generated from + // image.coords on each pass. + setImageRaw (catalog, Ncatalog, i, raw, Nraw, MODE_SIMPLE); + + NnewFit ++; + free (raw); + free (ref); + } + + fprintf (stderr, "UpdateStacksWithFit: %d fitted, %d keep old, %d skipped\n", (int) NnewFit, (int) NoldFit, (int) Nskip); + return (TRUE); +} + +// XXX 2 hardwired hacks in this file: 1) photcode hardwired for GPC1 stacks, 2) platescale Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/args.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/args.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/args.c (revision 40477) @@ -68,4 +68,11 @@ } + // the default is to only measure the scatter for the stacks, not to perform a new fit + FIT_STACKS = FALSE; + if ((N = get_argument (argc, argv, "-fit-stacks"))) { + remove_argument (N, &argc, argv); + FIT_STACKS = TRUE; + } + REPAIR_STACKS = FALSE; if ((N = get_argument (argc, argv, "-repair-stacks-on-update"))) { @@ -86,4 +93,11 @@ } + // only update the fit statistics (systematic floor) + IMSTATS_ONLY = FALSE; + if ((N = get_argument (argc, argv, "-imstats-only"))) { + remove_argument (N, &argc, argv); + IMSTATS_ONLY = TRUE; + } + // elements needed for parallel regions / parallel images MANUAL_UNIQUER = NULL; @@ -158,4 +172,9 @@ } + UPDATE_ALL_MEASURE = FALSE; + if ((N = get_argument (argc, argv, "-update-all-cameras"))) { + remove_argument (N, &argc, argv); + UPDATE_ALL_MEASURE = TRUE; + } UPDATE_PS1_STACK_MEASURE = FALSE; if ((N = get_argument (argc, argv, "-update-ps1-stack"))) { @@ -179,11 +198,17 @@ } if (RELASTRO_OP == OP_UPDATE_OFFSETS) { - if (!UPDATE_PS1_STACK_MEASURE && !UPDATE_PS1_CHIP_MEASURE && !UPDATE_HSC_MEASURE && !UPDATE_CFH_MEASURE) { - fprintf (stderr, "for -update-offsets, need to select at least one of -update-ps1-stack, -update-ps1-chip, -update-hsc, -update-cfh\n"); + if (!UPDATE_ALL_MEASURE && !UPDATE_PS1_STACK_MEASURE && !UPDATE_PS1_CHIP_MEASURE && !UPDATE_HSC_MEASURE && !UPDATE_CFH_MEASURE) { + fprintf (stderr, "for -update-offsets, need to select at least one of -update-all-cameras, -update-ps1-stack, -update-ps1-chip, -update-hsc, -update-cfh\n"); exit (2); } } - if ((RELASTRO_OP == OP_PARALLEL_IMAGES) || (RELASTRO_OP == OP_IMAGES)) { - if (APPLY_OFFSETS && !UPDATE_PS1_STACK_MEASURE && !UPDATE_PS1_CHIP_MEASURE && !UPDATE_HSC_MEASURE && !UPDATE_CFH_MEASURE) { + if (RELASTRO_OP == OP_UPDATE_OBJECTS) { + if (!UPDATE_ALL_MEASURE && !UPDATE_PS1_STACK_MEASURE && !UPDATE_PS1_CHIP_MEASURE && !UPDATE_HSC_MEASURE && !UPDATE_CFH_MEASURE) { + fprintf (stderr, "for -update-objects, need to select at least one of -update-all-cameras, -update-ps1-stack, -update-ps1-chip, -update-hsc, -update-cfh\n"); + exit (2); + } + } + if ((RELASTRO_OP == OP_PARALLEL_REGIONS) || (RELASTRO_OP == OP_PARALLEL_IMAGES) || (RELASTRO_OP == OP_IMAGES)) { + if (APPLY_OFFSETS && !UPDATE_ALL_MEASURE && !UPDATE_PS1_STACK_MEASURE && !UPDATE_PS1_CHIP_MEASURE && !UPDATE_HSC_MEASURE && !UPDATE_CFH_MEASURE) { fprintf (stderr, "for [-images or -parallel-images] with -apply-offsets, need to select at least one of -update-ps1-stack, -update-ps1-chip, -update-hsc, -update-cfh\n"); exit (2); @@ -300,4 +325,8 @@ remove_argument (N, &argc, argv); FIT_TARGET = SET_CHIPS; + } + if ((N = get_argument (argc, argv, "-set-stacks"))) { + remove_argument (N, &argc, argv); + FIT_TARGET = SET_STACKS; } @@ -858,4 +887,11 @@ } + // the default is to only measure the scatter for the stacks, not to perform a new fit + FIT_STACKS = FALSE; + if ((N = get_argument (argc, argv, "-fit-stacks"))) { + remove_argument (N, &argc, argv); + FIT_STACKS = TRUE; + } + REPAIR_STACKS = FALSE; if ((N = get_argument (argc, argv, "-repair-stacks-on-update"))) { @@ -869,4 +905,11 @@ } + // only update the fit statistics (systematic floor) + IMSTATS_ONLY = FALSE; + if ((N = get_argument (argc, argv, "-imstats-only"))) { + remove_argument (N, &argc, argv); + IMSTATS_ONLY = TRUE; + } + SKIP_PS1_CHIP = FALSE; if ((N = get_argument (argc, argv, "-skip-ps1-chip"))) { @@ -890,4 +933,9 @@ } + UPDATE_ALL_MEASURE = FALSE; + if ((N = get_argument (argc, argv, "-update-all-cameras"))) { + remove_argument (N, &argc, argv); + UPDATE_ALL_MEASURE = TRUE; + } UPDATE_PS1_STACK_MEASURE = FALSE; if ((N = get_argument (argc, argv, "-update-ps1-stack"))) { @@ -911,5 +959,5 @@ } if (RELASTRO_OP == OP_UPDATE_OFFSETS) { - if (!UPDATE_PS1_STACK_MEASURE && !UPDATE_PS1_CHIP_MEASURE && !UPDATE_HSC_MEASURE && !UPDATE_CFH_MEASURE) { + if (!UPDATE_ALL_MEASURE && !UPDATE_PS1_STACK_MEASURE && !UPDATE_PS1_CHIP_MEASURE && !UPDATE_HSC_MEASURE && !UPDATE_CFH_MEASURE) { fprintf (stderr, "for -update-offsets, need to select at least one of -update-ps1-stack, -update-ps1-chip, -update-hsc, -update-cfh\n"); exit (2); Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/fitexample.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/fitexample.c (revision 40477) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/fitexample.c (revision 40477) @@ -0,0 +1,349 @@ +# include "relastro.h" + +static int Nrand = 0; +double drand48_cnt () { + double value = drand48(); + Nrand ++; + return value; +} + +int mkstar (FitStats *fitStats, double Ro, double Do, double uR, double uD, double plx, int Npoints, int Nbad); + +# define MJD_MIN_MJD 55197 /* 2010/01/01,00:00:00 */ +# define MJD_MAX_MJD 57023 /* 2015/01/01,00:00:00 */ +# define MJD_MIN_YRS 10.00 /* 2010/01/01,00:00:00 */ +# define MJD_MAX_YRS 14.9993 /* 2015/01/01,00:00:00 */ +# define MJD_REF_YRS 12.4148 /* 2012/06/01,00:00:00 */ + +# define POS_ERROR 0.010 /* arcsec */ +# define OUT_ERROR 0.500 /* arcsec */ + +// # define N_STARS 3000 +// # define N_POINTS 100 +// # define N_OUTLIERS 5 +// # define N_BOOTSTRAP 100 + +# define dcos(THETA) cos(RAD_DEG*THETA) +# define dsin(THETA) sin(RAD_DEG*THETA) + +enum { + FIT_POS_NONE, + FIT_POS_IRLS, + FIT_POS_BOOT, + FIT_POS_NOCLIP, + FIT_PM_IRLS, + FIT_PM_BOOT, + FIT_PM_NOCLIP, + FIT_PLX_IRLS, + FIT_PLX_BOOT, + FIT_PLX_NOCLIP, +}; + +float ERROR_BIAS = 1.0; +float RA_BIAS = 0.0; +float DEC_BIAS = 0.0; + +int main (int argc, char **argv) { + + int N_STARS = 3000; + int N_POINTS = 100; + // int N_OUTLIERS = 10; + int N_BOOTSTRAP = 100; + float F_OUTLIERS = 0.1; + + int N; + if ((N = get_argument (argc, argv, "-Nstars"))) { + remove_argument (N, &argc, argv); + N_STARS = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-Npoints"))) { + remove_argument (N, &argc, argv); + N_POINTS = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + // if ((N = get_argument (argc, argv, "-Noutliers"))) { + // remove_argument (N, &argc, argv); + // N_OUTLIERS = atoi (argv[N]); + // remove_argument (N, &argc, argv); + // } + if ((N = get_argument (argc, argv, "-Nbootstrap"))) { + remove_argument (N, &argc, argv); + N_BOOTSTRAP = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-error-bias"))) { + remove_argument (N, &argc, argv); + ERROR_BIAS = atof (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-ra-bias"))) { + remove_argument (N, &argc, argv); + RA_BIAS = atof (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-dec-bias"))) { + remove_argument (N, &argc, argv); + DEC_BIAS = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc != 2) { + fprintf (stderr, "USAGE: %s (mode)\n", argv[0]); + fprintf (stderr, " mode: pos, pos-irls, pos-boot, pm, pm-irls, pm-boot, plx, plx-irls, plx-boot\n"); + fprintf (stderr, " options: -Nstars [3000], -Npoints [100], -Noutliers [10], -Nbootstrap [100]\n"); + exit (2); + } + + int mode = FIT_POS_NONE; + if (!strcasecmp(argv[1], "pos")) { + mode = FIT_POS_NOCLIP; + } + if (!strcasecmp(argv[1], "pos-irls")) { + mode = FIT_POS_IRLS; + } + if (!strcasecmp(argv[1], "pos-boot")) { + mode = FIT_POS_BOOT; + } + if (!strcasecmp(argv[1], "pm")) { + mode = FIT_PM_NOCLIP; + } + if (!strcasecmp(argv[1], "pm-irls")) { + mode = FIT_PM_IRLS; + } + if (!strcasecmp(argv[1], "pm-boot")) { + mode = FIT_PM_BOOT; + } + if (!strcasecmp(argv[1], "plx")) { + mode = FIT_PLX_NOCLIP; + } + if (!strcasecmp(argv[1], "plx-irls")) { + mode = FIT_PLX_IRLS; + } + if (!strcasecmp(argv[1], "plx-boot")) { + mode = FIT_PLX_BOOT; + } + + // plan_tests (14); + + // diag ("relastro fitpm tests"); + + // init the random seed + { + struct timeval now; + gettimeofday (&now, NULL); + long A = now.tv_sec + now.tv_usec * 1000000; + srand48(A); + srand48(1); + } + + ohana_gaussdev_init (); + + FitStats *fitStats = FitStatsInit (N_POINTS * (1 + 2*F_OUTLIERS), N_BOOTSTRAP); + // FitStats *fitStats = FitStatsInit (N_POINTS + N_OUTLIERS, N_BOOTSTRAP); + fitStats->fitdataPM->getChisq = TRUE; + fitStats->fitdataPM->getError = TRUE; + fitStats->fitdataPar->getChisq = TRUE; + fitStats->fitdataPar->getError = TRUE; + fitStats->fitdataPos->getChisq = TRUE; + fitStats->fitdataPos->getError = TRUE; + // NOTE: surprisingly, the chisq and error calculations only add ~2-3% to the exec time + + int i, k; + + int Nstars = N_STARS; + for (i = 0; i < Nstars; i++) { + + double Ro = 360.0*drand48_cnt(); + double Phi = 2.0*(drand48_cnt() - 0.5); + double Do = DEG_RAD * asin(Phi); + + double uR = 0.5*(drand48_cnt() - 0.5); + double uD = 0.5*(drand48_cnt() - 0.5); + + double plx = 0.5*(drand48_cnt() + 0.0); + + switch (mode) { + case FIT_POS_IRLS: + case FIT_POS_BOOT: + case FIT_POS_NOCLIP: + case FIT_PM_IRLS: + case FIT_PM_BOOT: + case FIT_PM_NOCLIP: + plx = 0.0; + break; + case FIT_PLX_IRLS: + case FIT_PLX_BOOT: + case FIT_PLX_NOCLIP: + break; + default: + myAbort("oops"); + } + + // int Npoints = N_POINTS*drand48_cnt(); // minimum of 0 points + // int Noutliers_max = Npoints * F_OUTLIERS; + // int Noutliers = Noutliers_max*drand48_cnt(); // minimum of 0 points + + int Npoints = N_POINTS; // minimum of 0 points + int Noutliers = Npoints * F_OUTLIERS; + + // generate a single fake star with N_POINTS real points and N_OUTLIERS bad points + mkstar (fitStats, Ro, Do, uR, uD, plx, Npoints, Noutliers); + + double Tmean, Trange, parRange; + FitAstromPoints_Project (fitStats, &Tmean, &Trange, &parRange); + + FitAstromResult fitResult; + FitAstromResultInit (&fitResult); + + int Nnomask; + + switch (mode) { + case FIT_POS_NOCLIP: + fitResult.uR = uR; + fitResult.uD = uD; + if (!FitPosPMfixed_Basic (&fitResult, fitStats->fitdataPos, fitStats->points, fitStats->Npoints)) goto escape; + break; + case FIT_POS_IRLS: + fitResult.uR = uR; + fitResult.uD = uD; + if (!FitPosPMfixed_IRLS (&fitResult, fitStats->fitdataPos, fitStats->points, fitStats->Npoints)) goto escape; + break; + case FIT_POS_BOOT: + fitResult.uR = uR; + fitResult.uD = uD; + if (!FitPosPMfixed_IRLS (&fitResult, fitStats->fitdataPos, fitStats->points, fitStats->Npoints)) goto escape; + fitStats->Nfit = 0; + Nnomask = BootstrapSaveUnmasked (fitStats->nomask, fitStats->points, fitStats->Npoints); + for (k = 0; k < fitStats->NfitAlloc; k++) { + BootstrapResample (fitStats->sample, fitStats->nomask, Nnomask); + fitStats->fit[k].uR = uR; + fitStats->fit[k].uD = uD; + if (!FitPosPMfixed_Basic (&fitStats->fit[fitStats->Nfit], fitStats->fitdataPos, fitStats->sample, Nnomask)) continue; + fitStats->Nfit ++; + } + // these calls set the ERRORS on the fit parameters, not the fit values (set in IRLS above) + BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_RA); + BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_DEC); + break; + + case FIT_PM_NOCLIP: + if (!FitPM_Basic (&fitResult, fitStats->fitdataPM, fitStats->points, fitStats->Npoints)) goto escape; + break; + case FIT_PM_IRLS: + if (!FitPM_IRLS (&fitResult, fitStats->fitdataPM, fitStats->points, fitStats->Npoints)) goto escape; + break; + case FIT_PM_BOOT: + if (!FitPM_IRLS (&fitResult, fitStats->fitdataPM, fitStats->points, fitStats->Npoints)) goto escape; + fitStats->Nfit = 0; + Nnomask = BootstrapSaveUnmasked (fitStats->nomask, fitStats->points, fitStats->Npoints); + for (k = 0; k < fitStats->NfitAlloc; k++) { + BootstrapResample (fitStats->sample, fitStats->nomask, Nnomask); + if (!FitPM_Basic (&fitStats->fit[fitStats->Nfit], fitStats->fitdataPM, fitStats->sample, Nnomask)) continue; + fitStats->Nfit ++; + } + // these calls set the ERRORS on the fit parameters, not the fit values (set in IRLS above) + BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_RA); + BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_DEC); + BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_uR); + BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_uD); + break; + + case FIT_PLX_NOCLIP: + if (!FitPMandPar_Basic (&fitResult, fitStats->fitdataPar, fitStats->points, fitStats->Npoints)) goto escape; + break; + case FIT_PLX_IRLS: + if (!FitPMandPar_IRLS (&fitResult, fitStats->fitdataPar, fitStats->points, fitStats->Npoints)) goto escape; + break; + case FIT_PLX_BOOT: + if (!FitPMandPar_IRLS (&fitResult, fitStats->fitdataPar, fitStats->points, fitStats->Npoints)) goto escape; + fitStats->Nfit = 0; + Nnomask = BootstrapSaveUnmasked (fitStats->nomask, fitStats->points, fitStats->Npoints); + for (k = 0; k < fitStats->NfitAlloc; k++) { + BootstrapResample (fitStats->sample, fitStats->nomask, Nnomask); + if (!FitPMandPar_Basic (&fitStats->fit[fitStats->Nfit], fitStats->fitdataPar, fitStats->sample, Nnomask)) continue; + fitStats->Nfit ++; + } + // these calls set the ERRORS on the fit parameters, not the fit values (set in IRLS above) + BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_RA); + BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_DEC); + BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_uR); + BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_uD); + BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_PLX); + break; + + default: + myAbort("oops"); + } + + XY_to_RD (&fitResult.Ro, &fitResult.Do, fitResult.Ro, fitResult.Do, &fitStats->coords); + + fprintf (stdout, "%12.8f %12.8f %8.6f %8.6f %8.6f | %12.8f %12.8f %8.6f %8.6f %8.6f | %12.8f | %8.6f %8.6f %8.6f %8.6f %8.6f %d | %f\n", + Ro, Do, uR, uD, plx, + fitResult.Ro, fitResult.Do, fitResult.uR, fitResult.uD, fitResult.p, Tmean, fitResult.dRo, fitResult.dDo, fitResult.duR, fitResult.duD, fitResult.dp, fitResult.Nfit, fitResult.chisq); + if (i > 0) { + // fprintf (stderr, ".\n"); + } + + continue; + + escape: + fprintf (stdout, "%12.8f %12.8f %8.6f %8.6f %8.6f | %12.8f %12.8f %8.6f %8.6f %8.6f | %12.8f | %8.6f %8.6f %8.6f %8.6f %8.6f %d | %f\n", + Ro, Do, uR, uD, plx, + NAN, NAN, NAN, NAN, NAN, 0.0, NAN, NAN, NAN, NAN, NAN, 0, 0.0); + } + // return exit_status(); + ohana_gaussdev_free(); + + exit (0); +} + +int mkstar (FitStats *fitStats, double Ro, double Do, double uR, double uD, double plx, int Npoints, int Nbad) { + + int i; + + int Ntotal = Npoints + Nbad; + + FitAstromPoint *points = fitStats->points; + + for (i = 0; i < Npoints; i++) { + FitAstromPointInit (&points[i]); + + // ParFactor expects a time which is in years since J2000 (MJD_..._YRS is so defined) + points[i].T = MJD_MIN_YRS + drand48_cnt()*(MJD_MAX_YRS - MJD_MIN_YRS); + + double pR, pD; + ParFactor (&pR, &pD, Ro, Do, points[i].T); + + double dR = ohana_gaussdev_rnd(0.0, POS_ERROR); + double dD = ohana_gaussdev_rnd(0.0, POS_ERROR); + + points[i].R = Ro + (dR + plx*pR + uR*(points[i].T - MJD_REF_YRS))/3600/dcos(Do); + points[i].D = Do + (dD + plx*pD + uD*(points[i].T - MJD_REF_YRS))/3600; + + points[i].dX = POS_ERROR*ERROR_BIAS; + points[i].dY = POS_ERROR*ERROR_BIAS; + } + + for (i = Npoints; i < Ntotal; i++) { + FitAstromPointInit (&points[i]); + + // ParFactor expects a time which is in years since J2000 (MJD_..._YRS is so defined) + points[i].T = MJD_MIN_YRS + drand48_cnt()*(MJD_MAX_YRS - MJD_MIN_YRS); + + double pR, pD; + ParFactor (&pR, &pD, Ro, Do, points[i].T); + + double dR = OUT_ERROR*(drand48_cnt() - 0.5) + RA_BIAS; + double dD = OUT_ERROR*(drand48_cnt() - 0.5) + DEC_BIAS; + + points[i].R = Ro + (dR + plx*pR + uR*(points[i].T - MJD_REF_YRS))/3600/dcos(Do); + points[i].D = Do + (dD + plx*pD + uD*(points[i].T - MJD_REF_YRS))/3600; + + points[i].dX = POS_ERROR*ERROR_BIAS; + points[i].dY = POS_ERROR*ERROR_BIAS; + } + fitStats->Npoints = Ntotal; + + return TRUE; +} Index: anches/czw_branch/20170908/Ohana/src/relastro/src/fitpm.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/fitpm.c (revision 40476) +++ (revision ) @@ -1,349 +1,0 @@ -# include "relastro.h" - -static int Nrand = 0; -double drand48_cnt () { - double value = drand48(); - Nrand ++; - return value; -} - -int mkstar (FitStats *fitStats, double Ro, double Do, double uR, double uD, double plx, int Npoints, int Nbad); - -# define MJD_MIN_MJD 55197 /* 2010/01/01,00:00:00 */ -# define MJD_MAX_MJD 57023 /* 2015/01/01,00:00:00 */ -# define MJD_MIN_YRS 10.00 /* 2010/01/01,00:00:00 */ -# define MJD_MAX_YRS 14.9993 /* 2015/01/01,00:00:00 */ -# define MJD_REF_YRS 12.4148 /* 2012/06/01,00:00:00 */ - -# define POS_ERROR 0.010 /* arcsec */ -# define OUT_ERROR 0.500 /* arcsec */ - -// # define N_STARS 3000 -// # define N_POINTS 100 -// # define N_OUTLIERS 5 -// # define N_BOOTSTRAP 100 - -# define dcos(THETA) cos(RAD_DEG*THETA) -# define dsin(THETA) sin(RAD_DEG*THETA) - -enum { - FIT_POS_NONE, - FIT_POS_IRLS, - FIT_POS_BOOT, - FIT_POS_NOCLIP, - FIT_PM_IRLS, - FIT_PM_BOOT, - FIT_PM_NOCLIP, - FIT_PLX_IRLS, - FIT_PLX_BOOT, - FIT_PLX_NOCLIP, -}; - -float ERROR_BIAS = 1.0; -float RA_BIAS = 0.0; -float DEC_BIAS = 0.0; - -int main (int argc, char **argv) { - - int N_STARS = 3000; - int N_POINTS = 100; - // int N_OUTLIERS = 10; - int N_BOOTSTRAP = 100; - float F_OUTLIERS = 0.1; - - int N; - if ((N = get_argument (argc, argv, "-Nstars"))) { - remove_argument (N, &argc, argv); - N_STARS = atoi (argv[N]); - remove_argument (N, &argc, argv); - } - if ((N = get_argument (argc, argv, "-Npoints"))) { - remove_argument (N, &argc, argv); - N_POINTS = atoi (argv[N]); - remove_argument (N, &argc, argv); - } - // if ((N = get_argument (argc, argv, "-Noutliers"))) { - // remove_argument (N, &argc, argv); - // N_OUTLIERS = atoi (argv[N]); - // remove_argument (N, &argc, argv); - // } - if ((N = get_argument (argc, argv, "-Nbootstrap"))) { - remove_argument (N, &argc, argv); - N_BOOTSTRAP = atoi (argv[N]); - remove_argument (N, &argc, argv); - } - if ((N = get_argument (argc, argv, "-error-bias"))) { - remove_argument (N, &argc, argv); - ERROR_BIAS = atof (argv[N]); - remove_argument (N, &argc, argv); - } - if ((N = get_argument (argc, argv, "-ra-bias"))) { - remove_argument (N, &argc, argv); - RA_BIAS = atof (argv[N]); - remove_argument (N, &argc, argv); - } - if ((N = get_argument (argc, argv, "-dec-bias"))) { - remove_argument (N, &argc, argv); - DEC_BIAS = atof (argv[N]); - remove_argument (N, &argc, argv); - } - - if (argc != 2) { - fprintf (stderr, "USAGE: %s (mode)\n", argv[0]); - fprintf (stderr, " mode: pos, pos-irls, pos-boot, pm, pm-irls, pm-boot, plx, plx-irls, plx-boot\n"); - fprintf (stderr, " options: -Nstars [3000], -Npoints [100], -Noutliers [10], -Nbootstrap [100]\n"); - exit (2); - } - - int mode = FIT_POS_NONE; - if (!strcasecmp(argv[1], "pos")) { - mode = FIT_POS_NOCLIP; - } - if (!strcasecmp(argv[1], "pos-irls")) { - mode = FIT_POS_IRLS; - } - if (!strcasecmp(argv[1], "pos-boot")) { - mode = FIT_POS_BOOT; - } - if (!strcasecmp(argv[1], "pm")) { - mode = FIT_PM_NOCLIP; - } - if (!strcasecmp(argv[1], "pm-irls")) { - mode = FIT_PM_IRLS; - } - if (!strcasecmp(argv[1], "pm-boot")) { - mode = FIT_PM_BOOT; - } - if (!strcasecmp(argv[1], "plx")) { - mode = FIT_PLX_NOCLIP; - } - if (!strcasecmp(argv[1], "plx-irls")) { - mode = FIT_PLX_IRLS; - } - if (!strcasecmp(argv[1], "plx-boot")) { - mode = FIT_PLX_BOOT; - } - - // plan_tests (14); - - // diag ("relastro fitpm tests"); - - // init the random seed - { - struct timeval now; - gettimeofday (&now, NULL); - long A = now.tv_sec + now.tv_usec * 1000000; - srand48(A); - srand48(1); - } - - ohana_gaussdev_init (); - - FitStats *fitStats = FitStatsInit (N_POINTS * (1 + 2*F_OUTLIERS), N_BOOTSTRAP); - // FitStats *fitStats = FitStatsInit (N_POINTS + N_OUTLIERS, N_BOOTSTRAP); - fitStats->fitdataPM->getChisq = TRUE; - fitStats->fitdataPM->getError = TRUE; - fitStats->fitdataPar->getChisq = TRUE; - fitStats->fitdataPar->getError = TRUE; - fitStats->fitdataPos->getChisq = TRUE; - fitStats->fitdataPos->getError = TRUE; - // NOTE: surprisingly, the chisq and error calculations only add ~2-3% to the exec time - - int i, k; - - int Nstars = N_STARS; - for (i = 0; i < Nstars; i++) { - - double Ro = 360.0*drand48_cnt(); - double Phi = 2.0*(drand48_cnt() - 0.5); - double Do = DEG_RAD * asin(Phi); - - double uR = 0.5*(drand48_cnt() - 0.5); - double uD = 0.5*(drand48_cnt() - 0.5); - - double plx = 0.5*(drand48_cnt() + 0.0); - - switch (mode) { - case FIT_POS_IRLS: - case FIT_POS_BOOT: - case FIT_POS_NOCLIP: - case FIT_PM_IRLS: - case FIT_PM_BOOT: - case FIT_PM_NOCLIP: - plx = 0.0; - break; - case FIT_PLX_IRLS: - case FIT_PLX_BOOT: - case FIT_PLX_NOCLIP: - break; - default: - myAbort("oops"); - } - - // int Npoints = N_POINTS*drand48_cnt(); // minimum of 0 points - // int Noutliers_max = Npoints * F_OUTLIERS; - // int Noutliers = Noutliers_max*drand48_cnt(); // minimum of 0 points - - int Npoints = N_POINTS; // minimum of 0 points - int Noutliers = Npoints * F_OUTLIERS; - - // generate a single fake star with N_POINTS real points and N_OUTLIERS bad points - mkstar (fitStats, Ro, Do, uR, uD, plx, Npoints, Noutliers); - - double Tmean, Trange, parRange; - FitAstromPoints_Project (fitStats, &Tmean, &Trange, &parRange); - - FitAstromResult fitResult; - FitAstromResultInit (&fitResult); - - int Nnomask; - - switch (mode) { - case FIT_POS_NOCLIP: - fitResult.uR = uR; - fitResult.uD = uD; - if (!FitPosPMfixed_Basic (&fitResult, fitStats->fitdataPos, fitStats->points, fitStats->Npoints)) goto escape; - break; - case FIT_POS_IRLS: - fitResult.uR = uR; - fitResult.uD = uD; - if (!FitPosPMfixed_IRLS (&fitResult, fitStats->fitdataPos, fitStats->points, fitStats->Npoints)) goto escape; - break; - case FIT_POS_BOOT: - fitResult.uR = uR; - fitResult.uD = uD; - if (!FitPosPMfixed_IRLS (&fitResult, fitStats->fitdataPos, fitStats->points, fitStats->Npoints)) goto escape; - fitStats->Nfit = 0; - Nnomask = BootstrapSaveUnmasked (fitStats->nomask, fitStats->points, fitStats->Npoints); - for (k = 0; k < fitStats->NfitAlloc; k++) { - BootstrapResample (fitStats->sample, fitStats->nomask, Nnomask); - fitStats->fit[k].uR = uR; - fitStats->fit[k].uD = uD; - if (!FitPosPMfixed_Basic (&fitStats->fit[fitStats->Nfit], fitStats->fitdataPos, fitStats->sample, Nnomask)) continue; - fitStats->Nfit ++; - } - // these calls set the ERRORS on the fit parameters, not the fit values (set in IRLS above) - BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_RA); - BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_DEC); - break; - - case FIT_PM_NOCLIP: - if (!FitPM_Basic (&fitResult, fitStats->fitdataPM, fitStats->points, fitStats->Npoints)) goto escape; - break; - case FIT_PM_IRLS: - if (!FitPM_IRLS (&fitResult, fitStats->fitdataPM, fitStats->points, fitStats->Npoints)) goto escape; - break; - case FIT_PM_BOOT: - if (!FitPM_IRLS (&fitResult, fitStats->fitdataPM, fitStats->points, fitStats->Npoints)) goto escape; - fitStats->Nfit = 0; - Nnomask = BootstrapSaveUnmasked (fitStats->nomask, fitStats->points, fitStats->Npoints); - for (k = 0; k < fitStats->NfitAlloc; k++) { - BootstrapResample (fitStats->sample, fitStats->nomask, Nnomask); - if (!FitPM_Basic (&fitStats->fit[fitStats->Nfit], fitStats->fitdataPM, fitStats->sample, Nnomask)) continue; - fitStats->Nfit ++; - } - // these calls set the ERRORS on the fit parameters, not the fit values (set in IRLS above) - BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_RA); - BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_DEC); - BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_uR); - BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_uD); - break; - - case FIT_PLX_NOCLIP: - if (!FitPMandPar_Basic (&fitResult, fitStats->fitdataPar, fitStats->points, fitStats->Npoints)) goto escape; - break; - case FIT_PLX_IRLS: - if (!FitPMandPar_IRLS (&fitResult, fitStats->fitdataPar, fitStats->points, fitStats->Npoints)) goto escape; - break; - case FIT_PLX_BOOT: - if (!FitPMandPar_IRLS (&fitResult, fitStats->fitdataPar, fitStats->points, fitStats->Npoints)) goto escape; - fitStats->Nfit = 0; - Nnomask = BootstrapSaveUnmasked (fitStats->nomask, fitStats->points, fitStats->Npoints); - for (k = 0; k < fitStats->NfitAlloc; k++) { - BootstrapResample (fitStats->sample, fitStats->nomask, Nnomask); - if (!FitPMandPar_Basic (&fitStats->fit[fitStats->Nfit], fitStats->fitdataPar, fitStats->sample, Nnomask)) continue; - fitStats->Nfit ++; - } - // these calls set the ERRORS on the fit parameters, not the fit values (set in IRLS above) - BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_RA); - BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_DEC); - BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_uR); - BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_uD); - BootstrapRobustStats (&fitResult, fitStats->fit, fitStats->Nfit, FIT_RESULT_PLX); - break; - - default: - myAbort("oops"); - } - - XY_to_RD (&fitResult.Ro, &fitResult.Do, fitResult.Ro, fitResult.Do, &fitStats->coords); - - fprintf (stdout, "%12.8f %12.8f %8.6f %8.6f %8.6f | %12.8f %12.8f %8.6f %8.6f %8.6f | %12.8f | %8.6f %8.6f %8.6f %8.6f %8.6f %d | %f\n", - Ro, Do, uR, uD, plx, - fitResult.Ro, fitResult.Do, fitResult.uR, fitResult.uD, fitResult.p, Tmean, fitResult.dRo, fitResult.dDo, fitResult.duR, fitResult.duD, fitResult.dp, fitResult.Nfit, fitResult.chisq); - if (i > 0) { - // fprintf (stderr, ".\n"); - } - - continue; - - escape: - fprintf (stdout, "%12.8f %12.8f %8.6f %8.6f %8.6f | %12.8f %12.8f %8.6f %8.6f %8.6f | %12.8f | %8.6f %8.6f %8.6f %8.6f %8.6f %d | %f\n", - Ro, Do, uR, uD, plx, - NAN, NAN, NAN, NAN, NAN, 0.0, NAN, NAN, NAN, NAN, NAN, 0, 0.0); - } - // return exit_status(); - ohana_gaussdev_free(); - - exit (0); -} - -int mkstar (FitStats *fitStats, double Ro, double Do, double uR, double uD, double plx, int Npoints, int Nbad) { - - int i; - - int Ntotal = Npoints + Nbad; - - FitAstromPoint *points = fitStats->points; - - for (i = 0; i < Npoints; i++) { - FitAstromPointInit (&points[i]); - - // ParFactor expects a time which is in years since J2000 (MJD_..._YRS is so defined) - points[i].T = MJD_MIN_YRS + drand48_cnt()*(MJD_MAX_YRS - MJD_MIN_YRS); - - double pR, pD; - ParFactor (&pR, &pD, Ro, Do, points[i].T); - - double dR = ohana_gaussdev_rnd(0.0, POS_ERROR); - double dD = ohana_gaussdev_rnd(0.0, POS_ERROR); - - points[i].R = Ro + (dR + plx*pR + uR*(points[i].T - MJD_REF_YRS))/3600/dcos(Do); - points[i].D = Do + (dD + plx*pD + uD*(points[i].T - MJD_REF_YRS))/3600; - - points[i].dX = POS_ERROR*ERROR_BIAS; - points[i].dY = POS_ERROR*ERROR_BIAS; - } - - for (i = Npoints; i < Ntotal; i++) { - FitAstromPointInit (&points[i]); - - // ParFactor expects a time which is in years since J2000 (MJD_..._YRS is so defined) - points[i].T = MJD_MIN_YRS + drand48_cnt()*(MJD_MAX_YRS - MJD_MIN_YRS); - - double pR, pD; - ParFactor (&pR, &pD, Ro, Do, points[i].T); - - double dR = OUT_ERROR*(drand48_cnt() - 0.5) + RA_BIAS; - double dD = OUT_ERROR*(drand48_cnt() - 0.5) + DEC_BIAS; - - points[i].R = Ro + (dR + plx*pR + uR*(points[i].T - MJD_REF_YRS))/3600/dcos(Do); - points[i].D = Do + (dD + plx*pD + uD*(points[i].T - MJD_REF_YRS))/3600; - - points[i].dX = POS_ERROR*ERROR_BIAS; - points[i].dY = POS_ERROR*ERROR_BIAS; - } - fitStats->Npoints = Ntotal; - - return TRUE; -} Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/launch_region_hosts.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/launch_region_hosts.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/launch_region_hosts.c (revision 40477) @@ -117,4 +117,7 @@ strextend (&command, "-set-chips"); break; + case SET_STACKS: + strextend (&command, "-set-stacks"); + break; case TARGET_NONE: abort(); @@ -151,4 +154,7 @@ if (SKIP_CFH) strextend (&command, "-skip-cfh"); + if (FIT_STACKS) strextend (&command, "-fit-stacks"); + if (IMSTATS_ONLY) strextend (&command, "-imstats-only"); + strextend (&command, "-nloop %d", NLOOP); strextend (&command, "-threads %d", NTHREADS); Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/relastro_images.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/relastro_images.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/relastro_images.c (revision 40477) @@ -99,5 +99,10 @@ UpdateChips (catalog, Ncatalog, 0); // measure.X,Y -> R,D, fit image.coords MARKTIME("update chips: %f sec\n", dtime); + break; + case SET_STACKS: + // we just want to fit the selected stacks to the mean positions + UpdateStacks (catalog, Ncatalog); + MARKTIME("update stacks : %f sec\n", dtime); break; Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/relastro_parallel_images.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/relastro_parallel_images.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/relastro_parallel_images.c (revision 40477) @@ -150,5 +150,5 @@ } - // measure scatter for stacks + // measure astrometry or scatter [default] for stacks UpdateStacks (catalog, Ncatalog); LOGRTIME("UpdateStacks on %s, host %d: %f sec\n", myHostName, REGION_HOST_ID, dtime); @@ -156,4 +156,14 @@ // create summary plots of the process // relastroVisualSummaryChips(); + break; + + case SET_STACKS: + // we just want to fit the selected stacks to the mean positions + share_mean_pos (catalog, Ncatalog, regionHosts, 0); + slurp_mean_pos (catalog, Ncatalog, regionHosts, 0); + UpdateStacks (catalog, Ncatalog); + share_image_pos (regionHosts, 0); + slurp_image_pos (catalog, Ncatalog, regionHosts, 0); + MARKTIME("update stacks : %f sec\n", dtime); break; Index: /branches/czw_branch/20170908/Ohana/src/relastro/src/select_images.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relastro/src/select_images.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relastro/src/select_images.c (revision 40477) @@ -114,4 +114,9 @@ } if (FALSE && !strncmp(timage[i].name, "o6227g0311o", 10)) { + fprintf (stderr, "test image 2\n"); + } + + // RINGS.V3.skycell.2634.034.stk.3411510.skycal.3811673.cmf[SkyChip.hdr] + if (!strncmp(timage[i].name, "RINGS.V3.skycell.2634.034.stk.3411510", strlen("RINGS.V3.skycell.2634.034.stk.3411510"))) { fprintf (stderr, "test image 2\n"); } Index: /branches/czw_branch/20170908/Ohana/src/relphot/include/relphot.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/include/relphot.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/include/relphot.h (revision 40477) @@ -77,11 +77,12 @@ unsigned int stop; short photcode; - float Mcal; + float McalPSF; + float McalAPER; float dMcal; float dMsys; - unsigned short nFitPhotom; - short Xm; + float McalChiSq; float secz; float ubercalDist; + unsigned short nFitPhotom; unsigned int flags; char skipCal; // if TRUE, this mosaic is incomplete and should not be calibrated @@ -201,8 +202,9 @@ typedef struct { - float Mcal; + float McalPSF; + float McalAPER; float dMcal; float dMagSys; - short Xm; + float McalChiSq; int nFitPhotom; int flags; @@ -220,5 +222,6 @@ typedef struct { - float Mcal; + float McalPSF; + float McalAPER; float dMcal; unsigned int imageID; @@ -291,4 +294,6 @@ int PARALLEL_SERIAL; char *MANUAL_UNIQUER; + +int SKIP_PARALLEL_GROUPS; int PARALLEL_REGIONS_MANUAL; @@ -336,4 +341,6 @@ int SyntheticPhotometry; +int USE_MCAL_PSF_FOR_STACK_APER; + char *PhotcodeList; @@ -404,4 +411,8 @@ int findMosaics PROTO((Catalog *catalog, int Ncatalog, int doMosaicList)); +void clearImages (void); +void checkImages (char *name); +int dumpMags (FILE *fout, Catalog *catalog, int Ncatalog); + void makeMosaics (Image *image, off_t Nimage, int mergeMcal); Mosaic *getMosaicForImage (off_t im); @@ -422,10 +433,9 @@ off_t getImageEntry PROTO((off_t meas, int cat)); -float getMcal_alt PROTO((off_t meas, int cat, FlatCorrectionTable *flatcorr, float Xccd, float Yccd)); -float getMcal PROTO((off_t meas, int cat, FlatCorrectionTable *flatcorr, Catalog *catalog)); +float getMcal PROTO((off_t meas, int cat, dvoMagClassType class)); float getMflat PROTO((off_t meas, int cat, FlatCorrectionTable *flatcorr, Catalog *catalog)); float getMgrid PROTO((off_t meas, int cat)); float getMmos PROTO((off_t meas, int cat)); -float getMrel PROTO((Catalog *catalog, off_t meas, int cat)); +float getMrel PROTO((Catalog *catalog, off_t meas, int cat, dvoMagClassType class, dvoMagSourceType source)); short getUbercalDist PROTO((off_t meas, int cat)); float getCenterOffset PROTO((off_t meas, int cat, Measure *measure, unsigned int *myID)); Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/BrightCatalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/BrightCatalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/BrightCatalog.c (revision 40477) @@ -66,5 +66,7 @@ GET_COLUMN(D, "DEC", double); GET_COLUMN(M, "MAG_SYS", float); - GET_COLUMN(Mcal, "MAG_CAL", float); + GET_COLUMN(Mkron, "MAG_KRON", float); + GET_COLUMN(McalPSF, "MCAL_PSF", float); + GET_COLUMN(McalAPER, "MCAL_APER", float); GET_COLUMN(Mflat, "MAG_FLAT", float); GET_COLUMN(dM, "MAG_ERR", float); @@ -89,5 +91,7 @@ measure[i].D = D[i]; measure[i].M = M[i]; - measure[i].Mcal = Mcal[i]; + measure[i].Mkron = Mkron[i]; + measure[i].McalPSF = McalPSF[i]; + measure[i].McalAPER = McalAPER[i]; measure[i].Mflat = Mflat[i]; measure[i].dM = dM[i]; @@ -110,5 +114,7 @@ free (D ); free (M ); - free (Mcal ); + free (Mkron ); + free (McalPSF ); + free (McalAPER); free (Mflat ); free (dM ); @@ -186,4 +192,5 @@ // need to create and assign to flat-field correction GET_COLUMN(M, "MAG", float); + GET_COLUMN(Mkron, "MAG_KRON", float); GET_COLUMN(dM, "MAG_ERR", float); GET_COLUMN(Mchisq, "MAG_CHI", float); @@ -199,16 +206,18 @@ ALLOCATE (secfilt, SecFilt, Nrow); for (i = 0; i < Nrow; i++) { - secfilt[i].M = M[i]; - secfilt[i].dM = dM[i]; - secfilt[i].Mchisq = Mchisq[i]; - secfilt[i].flags = flags[i]; - secfilt[i].Ncode = Ncode[i]; - secfilt[i].Nused = Nused[i]; - secfilt[i].Mmin = Mmin[i]; - secfilt[i].Mmax = Mmax[i]; + secfilt[i].MpsfChp = M[i]; + secfilt[i].MkronChp = Mkron[i]; + secfilt[i].dMpsfChp = dM[i]; + secfilt[i].Mchisq = Mchisq[i]; + secfilt[i].flags = flags[i]; + secfilt[i].Ncode = Ncode[i]; + secfilt[i].Nused = Nused[i]; + secfilt[i].Mmin = Mmin[i]; + secfilt[i].Mmax = Mmax[i]; } fprintf (stderr, "loaded data for %lld secfilt\n", (long long) Nrow); free (M ); + free (Mkron ); free (dM ); free (Mchisq); @@ -271,5 +280,7 @@ gfits_define_bintable_column (&theader, "D", "DEC", "dec", "degree", 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "MAG_SYS", "magnitude (sys)", NULL, 1.0, 0.0); - gfits_define_bintable_column (&theader, "E", "MAG_CAL", "magnitude (cal)", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_KRON", "magnitude (sys,kron)", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_PSF", "magnitude (cal)", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_APER","magnitude (cal)", NULL, 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "MAG_FLAT", "magnitude (flat)", NULL, 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "MAG_ERR", "magnitude (err)", NULL, 1.0, 0.0); @@ -293,5 +304,7 @@ double *D ; ALLOCATE (D , double, catalog->Nmeasure); float *M ; ALLOCATE (M , float, catalog->Nmeasure); - float *Mcal ; ALLOCATE (Mcal , float, catalog->Nmeasure); + float *Mkron ; ALLOCATE (Mkron , float, catalog->Nmeasure); + float *McalPSF ; ALLOCATE (McalPSF , float, catalog->Nmeasure); + float *McalAPER ; ALLOCATE (McalAPER , float, catalog->Nmeasure); float *Mflat ; ALLOCATE (Mflat , float, catalog->Nmeasure); float *dM ; ALLOCATE (dM , float, catalog->Nmeasure); @@ -311,8 +324,10 @@ MeasureTiny *measure = catalog->measure; for (i = 0; i < catalog->Nmeasure; i++) { - R[i] = measure[i].R ; - D[i] = measure[i].D ; + R[i] = measure[i].R ; + D[i] = measure[i].D ; M[i] = measure[i].M ; - Mcal[i] = measure[i].Mcal ; + Mkron[i] = measure[i].Mkron ; + McalPSF[i] = measure[i].McalPSF ; + McalAPER[i] = measure[i].McalAPER ; Mflat[i] = measure[i].Mflat ; dM[i] = measure[i].dM ; @@ -334,5 +349,7 @@ gfits_set_bintable_column (&theader, &ftable, "DEC", D, catalog->Nmeasure); gfits_set_bintable_column (&theader, &ftable, "MAG_SYS", M, catalog->Nmeasure); - gfits_set_bintable_column (&theader, &ftable, "MAG_CAL", Mcal, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MAG_KRON", Mkron, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MCAL_PSF", McalPSF, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MCAL_APER", McalAPER, catalog->Nmeasure); gfits_set_bintable_column (&theader, &ftable, "MAG_FLAT", Mflat, catalog->Nmeasure); gfits_set_bintable_column (&theader, &ftable, "MAG_ERR", dM, catalog->Nmeasure); @@ -352,5 +369,7 @@ free (D ); free (M ); - free (Mcal ); + free (Mkron ); + free (McalPSF ); + free (McalAPER); free (Mflat ); free (dM ); @@ -436,6 +455,7 @@ gfits_create_table_header (&theader, "BINTABLE", "SECFILT"); - gfits_define_bintable_column (&theader, "E", "MAG", "", "arcsec", 1.0, 0.0); - gfits_define_bintable_column (&theader, "E", "MAG_ERR", "", "arcsec", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG", "", "mag", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_KRON", "", "mag", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_ERR", "", "mag", 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "MAG_CHI", "", NULL, 1.0, 0.0); gfits_define_bintable_column (&theader, "J", "FLAGS", "", NULL, 1.0, 0.0); @@ -454,4 +474,5 @@ // create intermediate storage arrays float *M ; ALLOCATE (M , float, Nsec); + float *Mkron ; ALLOCATE (Mkron , float, Nsec); float *dM ; ALLOCATE (dM , float, Nsec); float *Mchisq ; ALLOCATE (Mchisq , float, Nsec); @@ -465,16 +486,18 @@ SecFilt *secfilt = catalog->secfilt; for (i = 0; i < Nsec; i++) { - M [i] = secfilt[i].M ; - dM [i] = secfilt[i].dM ; - Mchisq[i] = secfilt[i].Mchisq ; - flags [i] = secfilt[i].flags ; - Ncode [i] = secfilt[i].Ncode ; - Nused [i] = secfilt[i].Nused ; - Mmin [i] = secfilt[i].Mmin ; - Mmax [i] = secfilt[i].Mmax ; + M [i] = secfilt[i].MpsfChp ; + Mkron [i] = secfilt[i].MkronChp; + dM [i] = secfilt[i].dMpsfChp; + Mchisq[i] = secfilt[i].Mchisq ; + flags [i] = secfilt[i].flags ; + Ncode [i] = secfilt[i].Ncode ; + Nused [i] = secfilt[i].Nused ; + Mmin [i] = secfilt[i].Mmin ; + Mmax [i] = secfilt[i].Mmax ; } // add the columns to the output array gfits_set_bintable_column (&theader, &ftable, "MAG", M , Nsec); + gfits_set_bintable_column (&theader, &ftable, "MAG_KRON", Mkron , Nsec); gfits_set_bintable_column (&theader, &ftable, "MAG_ERR", dM , Nsec); gfits_set_bintable_column (&theader, &ftable, "MAG_CHI", Mchisq, Nsec); @@ -486,4 +509,5 @@ free (M ); + free (Mkron ); free (dM ); free (Mchisq ); Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/GridOps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/GridOps.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/GridOps.c (revision 40477) @@ -401,5 +401,5 @@ // skip images marked as BAD - Mcal = getMcal (m, c, flatcorr, catalog); + Mcal = getMcal (m, c, MAG_CLASS_PSF); if (isnan(Mcal)) { Ncal ++; @@ -558,5 +558,5 @@ continue; } - Mcal = getMcal (m, c, flatcorr, catalog); + Mcal = getMcal (m, c, MAG_CLASS_PSF); if (isnan(Mcal)) { Ncal ++; @@ -568,5 +568,5 @@ continue; } - Mrel = getMrel (catalog, m, c); + Mrel = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); if (isnan(Mrel)) { Nrel ++; @@ -645,9 +645,9 @@ continue; } - Mcal = getMcal (m, c, flatcorr, catalog); + Mcal = getMcal (m, c, MAG_CLASS_PSF); if (isnan(Mcal)) continue; Mmos = getMmos (m, c); if (isnan(Mmos)) continue; - Mrel = getMrel (catalog, m, c); + Mrel = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); if (isnan(Mrel)) continue; Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/ImageMagIO.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/ImageMagIO.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/ImageMagIO.c (revision 40477) @@ -58,12 +58,13 @@ char type[16]; - GET_COLUMN (Mcal, "MCAL", float); + GET_COLUMN (McalPSF, "MCAL_PSF", float); + GET_COLUMN (McalAPER, "MCAL_APER", float); GET_COLUMN (dMcal, "MCAL_ERR", float); GET_COLUMN (dMagSys, "MCAL_SYSERR", float); - GET_COLUMN (nFitPhotom, "NFIT", int); - GET_COLUMN (flags, "FLAGS", int); - GET_COLUMN (ubercalDist, "UDIST", int); - GET_COLUMN (imageID, "ID", int); - GET_COLUMN (Xm, "CHISQ", short); + GET_COLUMN (McalChiSq, "MCAL_CHISQ", float); + GET_COLUMN (nFitPhotom, "NFIT", int); + GET_COLUMN (flags, "FLAGS", int); + GET_COLUMN (ubercalDist, "UDIST", int); + GET_COLUMN (imageID, "ID", int); // free the memory associated with the FITS files @@ -75,8 +76,9 @@ ALLOCATE (image_mags, ImageMag, Nrow); for (i = 0; i < Nrow; i++) { - image_mags[i].Mcal = Mcal [i]; + image_mags[i].McalPSF = McalPSF [i]; + image_mags[i].McalAPER = McalAPER [i]; image_mags[i].dMcal = dMcal [i]; image_mags[i].dMagSys = dMagSys [i]; - image_mags[i].Xm = Xm [i]; + image_mags[i].McalChiSq = McalChiSq [i]; image_mags[i].nFitPhotom = nFitPhotom [i]; image_mags[i].flags = flags [i]; @@ -86,8 +88,9 @@ fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow); - free (Mcal ); + free (McalPSF ); + free (McalAPER ); free (dMcal ); free (dMagSys ); - free (Xm ); + free (McalChiSq ); free (nFitPhotom ); free (flags ); @@ -121,20 +124,22 @@ gfits_create_table_header (&theader, "BINTABLE", "IMAGE_MAGS"); - gfits_define_bintable_column (&theader, "E", "MCAL", "cal offset", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_PSF", "PSF cal offset", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_APER", "APER cal offset", "magnitudes", 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "MCAL_ERR", "cal error", "magnitudes", 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "MCAL_SYSERR", "systematic error", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_CHISQ", "cal chisq", "unitless", 1.0, 0.0); gfits_define_bintable_column (&theader, "J", "NFIT", "number of fitted stars", "unitless", 1.0, 0.0); gfits_define_bintable_column (&theader, "J", "FLAGS", "analysis flags", "unitless", 1.0, 0.0); gfits_define_bintable_column (&theader, "J", "UDIST", "distance to ubercal images", "images", 1.0, 0.0); gfits_define_bintable_column (&theader, "J", "ID", "image ID", "unitless", 1.0, 0.0); - gfits_define_bintable_column (&theader, "I", "CHISQ", "cal chisq", "unitless", 1.0, FT_BZERO_INT16); // generate the output array that carries the data gfits_create_table (&theader, &ftable); - float *Mcal ; + float *McalPSF ; + float *McalAPER ; float *dMcal ; float *dMagSys ; - float *Xm ; + float *McalChiSq ; int *nFitPhotom ; int *flags ; @@ -143,8 +148,9 @@ // create intermediate storage arrays - ALLOCATE (Mcal , float, Nimage_mags); + ALLOCATE (McalPSF , float, Nimage_mags); + ALLOCATE (McalAPER , float, Nimage_mags); ALLOCATE (dMcal , float, Nimage_mags); ALLOCATE (dMagSys , float, Nimage_mags); - ALLOCATE (Xm , float, Nimage_mags); + ALLOCATE (McalChiSq , float, Nimage_mags); ALLOCATE (nFitPhotom , int, Nimage_mags); ALLOCATE (flags , int, Nimage_mags); @@ -154,8 +160,9 @@ // assign the storage arrays for (i = 0; i < Nimage_mags; i++) { - Mcal [i] = image_mags[i].Mcal ; + McalPSF [i] = image_mags[i].McalPSF ; + McalAPER [i] = image_mags[i].McalAPER ; dMcal [i] = image_mags[i].dMcal ; dMagSys [i] = image_mags[i].dMagSys ; - Xm [i] = image_mags[i].Xm ; + McalChiSq [i] = image_mags[i].McalChiSq ; nFitPhotom [i] = image_mags[i].nFitPhotom ; flags [i] = image_mags[i].flags ; @@ -165,8 +172,9 @@ // add the columns to the output array - gfits_set_bintable_column (&theader, &ftable, "MCAL", Mcal , Nimage_mags); + gfits_set_bintable_column (&theader, &ftable, "MCAL_PSF", McalPSF , Nimage_mags); + gfits_set_bintable_column (&theader, &ftable, "MCAL_APER", McalAPER , Nimage_mags); gfits_set_bintable_column (&theader, &ftable, "MCAL_ERR", dMcal , Nimage_mags); gfits_set_bintable_column (&theader, &ftable, "MCAL_SYSERR", dMagSys , Nimage_mags); - gfits_set_bintable_column (&theader, &ftable, "CHISQ", Xm , Nimage_mags); + gfits_set_bintable_column (&theader, &ftable, "MCAL_CHISQ", McalChiSq , Nimage_mags); gfits_set_bintable_column (&theader, &ftable, "NFIT", nFitPhotom , Nimage_mags); gfits_set_bintable_column (&theader, &ftable, "FLAGS", flags , Nimage_mags); @@ -174,8 +182,9 @@ gfits_set_bintable_column (&theader, &ftable, "ID", imageID , Nimage_mags); - free (Mcal ); + free (McalPSF ); + free (McalAPER ); free (dMcal ); free (dMagSys ); - free (Xm ); + free (McalChiSq ); free (nFitPhotom ); free (flags ); Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/ImageOps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/ImageOps.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/ImageOps.c (revision 40477) @@ -143,5 +143,6 @@ image[i].photom_map_id = input[i].photom_map_id; image[i].flags = input[i].flags ; - image[i].Mcal = input[i].Mcal ; + image[i].McalPSF = input[i].McalPSF ; + image[i].McalAPER = input[i].McalAPER ; image[i].dMcal = input[i].dMcal ; image[i].tzero = input[i].tzero ; @@ -179,5 +180,6 @@ subset[i].photom_map_id = image[i].photom_map_id; subset[i].flags = image[i].flags ; - subset[i].Mcal = image[i].Mcal ; + subset[i].McalPSF = image[i].McalPSF ; + subset[i].McalAPER = image[i].McalAPER ; subset[i].dMcal = image[i].dMcal ; subset[i].tzero = image[i].tzero ; @@ -430,58 +432,27 @@ } -// returns image.Mcal +// returns image.McalPSF or image.McalAPER // NOTE: static flat-field component is included in measure.Mflat -float getMcal (off_t meas, int cat, FlatCorrectionTable *flatcorr, Catalog *catalog) { - OHANA_UNUSED_PARAM(flatcorr); - OHANA_UNUSED_PARAM(catalog); - - off_t i; - float value; +float getMcal (off_t meas, int cat, dvoMagClassType class) { + + off_t i; i = MeasureToImage[cat][meas]; - if (i == -1) return (NAN); - - if (image[i].flags & IMAGE_BAD) return (NAN); - value = image[i].Mcal; - - // to do this, I need to pass in the catalog and flatcorr pointers - // int flat_id = image[i].photom_map_id; - // float offset = 0.0; - // if (flat_id) { - // offset = FlatCorrectionOffset (flatcorr, flat_id, catalog[cat].measureT[meas].Xccd, catalog[cat].measureT[meas].Yccd); - // } - // value -= offset; - - return (value); -} - -// returns image.Mcal - ff(x,y) -// NOTE: static flat-field component is included in measure.Mflat -float getMcal_alt (off_t meas, int cat, FlatCorrectionTable *flatcorr, float Xccd, float Yccd) { - OHANA_UNUSED_PARAM(flatcorr); - OHANA_UNUSED_PARAM(Xccd); - OHANA_UNUSED_PARAM(Yccd); - - off_t i; - float value; - - i = MeasureToImage[cat][meas]; - if (i == -1) return (NAN); - - if (image[i].flags & IMAGE_BAD) return (NAN); - value = image[i].Mcal; - - // to do this, I need to pass in the catalog and flatcorr pointers - // int flat_id = image[i].photom_map_id; - // float offset = 0.0; - // if (flat_id) { - // offset = FlatCorrectionOffset (flatcorr, flat_id, Xccd, Yccd); - // } - // value -= offset; - - return (value); -} - -// returns image.Mcal - ff(x,y) + if (i == -1) return NAN; + + if (image[i].flags & IMAGE_BAD) return NAN; + + switch (class) { + case MAG_CLASS_PSF: + return image[i].McalPSF; + case MAG_CLASS_APER: + case MAG_CLASS_KRON: + return image[i].McalAPER; + default: + return NAN; + } + return NAN; // should not be able to reach here +} + short getUbercalDist (off_t meas, int cat) { @@ -499,5 +470,4 @@ } -// returns image.Mcal - ff(x,y) float getCenterOffset (off_t meas, int cat, Measure *measure, unsigned int *myID) { @@ -519,5 +489,4 @@ } -// returns image.Mcal - ff(x,y) int MatchImageName (off_t meas, int cat, char *name) { @@ -542,5 +511,4 @@ } -// returns image.Mcal - ff(x,y) int MatchImageSkycellID (off_t meas, int cat, int myTessID, int myProjectionID, int mySkycellID) { @@ -563,5 +531,4 @@ } -// returns image.Mcal - ff(x,y) int FindImageSkycellID (off_t meas, int cat, int *myTessID, int *myProjectionID, int *mySkycellID) { @@ -618,8 +585,6 @@ void setMcal (Catalog *catalog, int PoorImages, FlatCorrectionTable *flatcorr) { - off_t i, j, m, c, n, Nmax; + off_t i, j, m, c, n; int mark, bad, Nfew, Nbad, Nmos, Nrel, Ngrid, Nsys; - float Msys, Mrel, Mmos, Mgrid, Mflat; - // double *list, *dlist, *Mlist, *dMlist; StatType stats; @@ -638,9 +603,11 @@ } - Nmax = 0; + off_t Nmax = 0; for (i = 0; i < Nimage; i++) { Nmax = MAX (Nmax, N_onImage[i]); } - StatDataSet *refStars = StatDataSetAlloc (1, Nmax); + + StatDataSet *kronStars = StatDataSetAlloc (1, Nmax); + StatDataSet *psfStars = StatDataSetAlloc (1, Nmax); StatDataSet *brightStars = StatDataSetAlloc (1, Nmax); @@ -676,6 +643,7 @@ int minUbercalDist = 1000; - off_t Nref = 0; // total number of reference stars on the image - int Nbright = 0; // number of stars to measure the bright-end scatter + off_t Nref = 0; // number of stars used to measure McalPSF + int Nkron = 0; // number of stars to measure McalAPER + int Nbright = 0; // number of stars to measure the bright-end scatter if (N_onImage[i] == 0) { @@ -693,16 +661,23 @@ continue; } - Mmos = getMmos (m, c); + float Mmos = getMmos (m, c); if (isnan(Mmos)) { Nmos ++; continue; } - Mgrid = getMgrid (m, c); + float Mgrid = getMgrid (m, c); if (isnan(Mgrid)) { Ngrid++; continue; } - Mrel = getMrel (catalog, m, c); - if (isnan(Mrel)) { + + // Mrel* is the average magnitude for this star. For PS1 stacks, we have too much + // PSF variability. We need to calibrate the PSF magnitudes separately from the + // Aperture-like magnitues. (We have an option to use the kron magnitudes or the + // other apertures here). I basically need to do this analysis separately for each + // magnitude type + + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + if (isnan(MrelPSF)) { Nrel ++; continue; @@ -714,9 +689,9 @@ // the flat-correction. NOTE the sign of Mflat (Image.Mcal = Measure.Mcal - Mflat) - Mflat = getMflat (m, c, flatcorr, catalog); + float Mflat = getMflat (m, c, flatcorr, catalog); n = catalog[c].measureT[m].averef; - Msys = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); - if (isnan(Msys)) { + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + if (isnan(MsysPSF)) { Nsys++; continue; @@ -731,19 +706,30 @@ skip: - refStars->flxlist[Nref] = Msys - Mrel - Mmos - Mgrid + Mflat; - refStars->errlist[Nref] = MAX (catalog[c].measureT[m].dM, MIN_ERROR); - refStars->wgtlist[Nref] = 1; - refStars->msklist[Nref] = 0; - if (fabs(refStars->flxlist[Nref]) > 0.03) { + psfStars->flxlist[Nref] = MsysPSF - MrelPSF - Mmos - Mgrid + Mflat; + psfStars->errlist[Nref] = MAX (catalog[c].measureT[m].dM, MIN_ERROR); + psfStars->wgtlist[Nref] = 1; + psfStars->msklist[Nref] = 0; + if (fabs(psfStars->flxlist[Nref]) > 0.03) { // fprintf (stderr, "deviant\n"); } + float MrelKron = getMrel (catalog, m, c, MAG_CLASS_KRON, MAG_SRC_CHP); + float MsysKron = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_KRON); + + if (isfinite(MrelKron) && isfinite(MsysKron)) { + kronStars->flxlist[Nkron] = MsysKron - MrelKron - Mmos - Mgrid + Mflat; + kronStars->errlist[Nkron] = MAX (catalog[c].measureT[m].dM, MIN_ERROR); + kronStars->wgtlist[Nkron] = 1; + kronStars->msklist[Nkron] = 0; + Nkron ++; + } + if ((image[i].imageID == TEST_IMAGE1) || (image[i].imageID == TEST_IMAGE2)) { - fprintf (stderr, "%1d, %3d : %3d, %3d : %10.6f %10.6f : %6.3f %6.3f %6.3f %6.3f %6.3f : %6.3f\n", (int) i, (int) j, (int) c, (int) m, catalog[c].averageT[n].R, catalog[c].averageT[n].D, Msys, Mrel, Mmos, Mgrid, Mflat, refStars->flxlist[Nref]); + fprintf (stderr, "%1d, %3d : %3d, %3d : %10.6f %10.6f : %6.3f %6.3f %6.3f %6.3f %6.3f : %6.3f\n", (int) i, (int) j, (int) c, (int) m, catalog[c].averageT[n].R, catalog[c].averageT[n].D, MsysKron, MrelKron, Mmos, Mgrid, Mflat, kronStars->flxlist[Nref]); } if (catalog[c].measureT[m].dM < IMFIT_SYS_SIGMA_LIM) { - brightStars->flxlist[Nbright] = refStars->flxlist[Nref]; - brightStars->errlist[Nbright] = refStars->errlist[Nref]; + brightStars->flxlist[Nbright] = psfStars->flxlist[Nref]; + brightStars->errlist[Nbright] = psfStars->errlist[Nref]; brightStars->wgtlist[Nbright] = 1; brightStars->msklist[Nbright] = 0; @@ -770,31 +756,39 @@ // no additional weight modification (we treat all stars on an image equally -- note an image is either ubercal-tied or not) # if (BASIC_STATS) - liststats (refStars->flxlist, refStars->errlist, NULL, Nref, &stats); + liststats (psfStars->flxlist, psfStars->errlist, NULL, Nref, &stats); # else - liststats_irls (refStars, Nref, &stats); + liststats_irls (psfStars, Nref, &stats); # endif - image[i].Mcal = stats.mean; + image[i].McalPSF = stats.mean; image[i].dMcal = stats.error; image[i].nFitPhotom = Nref; - image[i].Xm = 100.0*log10(stats.chisq); + image[i].McalChiSq = stats.chisq; Ncalibrated ++; + + // no additional weight modification (we treat all stars on an image equally -- note an image is either ubercal-tied or not) +# if (BASIC_STATS) + liststats (kronStars->flxlist, kronStars->errlist, NULL, Nkron, &stats); +# else + liststats_irls (kronStars, Nref, &stats); +# endif + image[i].McalAPER = stats.mean; if ((image[i].imageID == TEST_IMAGE1) || (image[i].imageID == TEST_IMAGE2)) { for (j = 0; j < Nref; j++) { - fprintf (stderr, "%1d, %8d : %6.3f %6.3f %6.3f %d\n", (int) i, (int) image[i].imageID, refStars->flxlist[j], refStars->errlist[j], refStars->wgtlist[j], refStars->msklist[j]); + fprintf (stderr, "%1d, %8d : %6.3f %6.3f %6.3f %d\n", (int) i, (int) image[i].imageID, kronStars->flxlist[j], kronStars->errlist[j], kronStars->wgtlist[j], kronStars->msklist[j]); } } if ((image[i].imageID == TEST_IMAGE1) || (image[i].imageID == TEST_IMAGE2)) { - fprintf (stderr, "Mcal for : %s : %7.4f %7.4f\n", image[i].name, image[i].Mcal, image[i].dMcal); + fprintf (stderr, "Mcal for : %s : %7.4f %7.4f\n", image[i].name, image[i].McalAPER, image[i].dMcal); } if (!mark && VERBOSE_IMAGE) { - fprintf (stderr, "Mcal for : %s : %7.4f %7.4f\n", image[i].name, image[i].Mcal, image[i].dMcal); + fprintf (stderr, "Mcal for : %s : %7.4f %7.4f\n", image[i].name, image[i].McalAPER, image[i].dMcal); } if (PLOTSTUFF) { - fprintf (stderr, "Mcal for : %s : %7.4f %7.4f\n", image[i].name, image[i].Mcal, image[i].dMcal); - plot_setMcal (refStars->flxlist, Nref, &stats, CLOUD_TOLERANCE); + fprintf (stderr, "Mcal for : %s : %7.4f %7.4f\n", image[i].name, image[i].McalAPER, image[i].dMcal); + plot_setMcal (psfStars->flxlist, Nref, &stats, CLOUD_TOLERANCE); } @@ -803,6 +797,6 @@ image[i].dMagSys = stats.sigma; - if (image[i].Mcal < -CLOUD_TOLERANCE) { - image[i].Mcal = 0.0; + if (image[i].McalPSF < -CLOUD_TOLERANCE) { + image[i].McalPSF = 0.0; } @@ -813,5 +807,6 @@ StatDataSetFree (brightStars, 1); - StatDataSetFree (refStars, 1); + StatDataSetFree (kronStars, 1); + StatDataSetFree (psfStars, 1); fprintf (stderr, "%d images calibrated\n", Ncalibrated); @@ -848,5 +843,5 @@ if (FREEZE_IMAGES && isGPC1chip(image[i].photcode)) continue; - mlist[N] = image[i].Mcal; + mlist[N] = image[i].McalPSF; slist[N] = image[i].dMcal; dlist[N] = 1; @@ -873,5 +868,5 @@ mark = FALSE; image[i].flags &= ~ID_IMAGE_PHOTOM_POOR; - mark = (image[i].dMcal > MaxScatter) || (fabs(image[i].Mcal - MedOffset) > MaxOffset); + mark = (image[i].dMcal > MaxScatter) || (fabs(image[i].McalPSF - MedOffset) > MaxOffset); if (mark) { Nmark ++; @@ -961,11 +956,11 @@ if (FREEZE_IMAGES && isGPC1chip(image[i].photcode)) continue; - Mlist[Nplot] = image[i].Mcal; + Mlist[Nplot] = image[i].McalPSF; dlist[Nplot] = image[i].dMcal; xlist[Nplot] = image[i].secz; minAirmass = MIN (image[i].secz, minAirmass); maxAirmass = MAX (image[i].secz, maxAirmass); - minMcal = MIN (image[i].Mcal, minMcal); - maxMcal = MAX (image[i].Mcal, maxMcal); + minMcal = MIN (image[i].McalPSF, minMcal); + maxMcal = MAX (image[i].McalPSF, maxMcal); mindMcal = MIN (image[i].dMcal, mindMcal); maxdMcal = MAX (image[i].dMcal, maxdMcal); @@ -1051,5 +1046,5 @@ c = ImageToCatalog[i][j]; - Mcal = getMcal (m, c, flatcorr, catalog); + Mcal = getMcal (m, c, MAG_CLASS_PSF); if (isnan(Mcal)) continue; Mmos = getMmos (m, c); @@ -1092,5 +1087,5 @@ if (FREEZE_IMAGES && isGPC1chip(image[i].photcode)) continue; - list[n] = pow (10.0, 0.01*image[i].Xm); + list[n] = image[i].McalChiSq; dlist[n] = 1; n++; @@ -1125,5 +1120,5 @@ if (FREEZE_IMAGES && isGPC1chip(image[i].photcode)) continue; - list[n] = image[i].Mcal; + list[n] = image[i].McalPSF; dlist[n] = 1; n++; @@ -1170,2 +1165,21 @@ return (stats); } + +void clearImages (void) { + image = NULL; +} + +void checkImages (char *name) { + + off_t i; + fprintf (stderr, "--- %s ---\n", name); + + OhanaMemblock *ref = (OhanaMemblock *) image - 1; + fprintf (stderr, "file: %s\n", ref->file); + + fprintf (stderr, "0x%08llx \n", (unsigned long long) image); + + for (i = 0; i < Nimage; i++) { + fprintf (stderr, "%s : 0x%08x : %6.3f %6.3f\n", image[i].name, image[i].flags, image[i].McalPSF, image[i].McalAPER); + } +} Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/ImageSubset.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/ImageSubset.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/ImageSubset.c (revision 40477) @@ -63,5 +63,6 @@ char type[16]; - GET_COLUMN (Mcal, "MCAL", float); + GET_COLUMN (McalPSF, "MCAL_PSF", float); + GET_COLUMN (McalAPER, "MCAL_APER", float); GET_COLUMN (dMcal, "MCAL_ERR", float); GET_COLUMN (imageID, "IMAGE_ID", int); @@ -79,5 +80,6 @@ ALLOCATE (image, ImageSubset, Nrow); for (i = 0; i < Nrow; i++) { - image[i].Mcal = Mcal[i]; + image[i].McalPSF = McalPSF[i]; + image[i].McalAPER = McalAPER[i]; image[i].dMcal = dMcal[i]; image[i].imageID = imageID[i]; @@ -93,5 +95,6 @@ fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow); - free (Mcal); + free (McalPSF); + free (McalAPER); free (dMcal); free (imageID); @@ -136,5 +139,6 @@ gfits_create_table_header (&theader, "BINTABLE", "IMAGE_SUBSET"); - gfits_define_bintable_column (&theader, "E", "MCAL", "zero point offset", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_PSF", "zero point offset", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_APER", "zero point offset", "magnitudes", 1.0, 0.0); gfits_define_bintable_column (&theader, "E", "MCAL_ERR", "zero point error", "magnitudes", 1.0, 0.0); gfits_define_bintable_column (&theader, "J", "IMAGE_ID", "image ID", NULL, 1.0, FT_BZERO_INT32); @@ -151,5 +155,5 @@ gfits_create_table (&theader, &ftable); - float *Mcal, *dMcal; + float *McalPSF, *McalAPER, *dMcal; unsigned int *imageID, *map, *flags, *tzero; int *tessID, *projID, *skycellID; @@ -158,5 +162,6 @@ // create intermediate storage arrays - ALLOCATE (Mcal, float, Nimage); + ALLOCATE (McalPSF, float, Nimage); + ALLOCATE (McalAPER, float, Nimage); ALLOCATE (dMcal, float, Nimage); ALLOCATE (imageID, unsigned int, Nimage); @@ -172,5 +177,6 @@ // assign the storage arrays for (i = 0; i < Nimage; i++) { - Mcal[i] = image[i].Mcal; + McalPSF[i] = image[i].McalPSF; + McalAPER[i] = image[i].McalAPER; dMcal[i] = image[i].dMcal; imageID[i] = image[i].imageID; @@ -186,5 +192,6 @@ // add the columns to the output array - gfits_set_bintable_column (&theader, &ftable, "MCAL", Mcal, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MCAL_PSF", McalPSF, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MCAL_APER", McalAPER, Nimage); gfits_set_bintable_column (&theader, &ftable, "MCAL_ERR", dMcal, Nimage); gfits_set_bintable_column (&theader, &ftable, "IMAGE_ID", imageID, Nimage); @@ -198,5 +205,6 @@ gfits_set_bintable_column (&theader, &ftable, "UBERCAL_DIST", ucdist, Nimage); - free (Mcal); + free (McalPSF); + free (McalAPER); free (dMcal); free (imageID); Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/MosaicOps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/MosaicOps.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/MosaicOps.c (revision 40477) @@ -111,13 +111,14 @@ /* a new mosaic, define ranges -- preserve the original values incase this image is not used */ - mosaic[Nmosaic].start = start; - mosaic[Nmosaic].stop = stop; - mosaic[Nmosaic].Mcal = 0.0; // note : mosaic stores only offsets relative to the original image values - mosaic[Nmosaic].dMcal = 0.0; // note : at the end, mosaic.Mcal is added back to the input images - mosaic[Nmosaic].dMsys = 0.0; - mosaic[Nmosaic].Xm = 0.0; - mosaic[Nmosaic].flags = image[i].flags; - mosaic[Nmosaic].secz = image[i].secz; - mosaic[Nmosaic].photcode = GetPhotcodeEquivCodebyCode (image[i].photcode); + mosaic[Nmosaic].start = start; + mosaic[Nmosaic].stop = stop; + mosaic[Nmosaic].McalPSF = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[Nmosaic].McalAPER = 0.0; // note : mosaic stores only offsets relative to the original image values + mosaic[Nmosaic].dMcal = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[Nmosaic].dMsys = 0.0; + mosaic[Nmosaic].McalChiSq = 0.0;// NAN or 0.0? + mosaic[Nmosaic].flags = image[i].flags; + mosaic[Nmosaic].secz = image[i].secz; + mosaic[Nmosaic].photcode = GetPhotcodeEquivCodebyCode (image[i].photcode); // XXX do we need to do something about flag consistency across a mosaic? @@ -208,14 +209,15 @@ for (i = 0; i < Nmosaic; i++) { /* a new mosaic, define ranges */ - mosaic[i].start = startTimesMosaic[i]; - mosaic[i].stop = 0; - mosaic[i].Mcal = 0.0; - mosaic[i].dMcal = 0.0; - mosaic[i].dMsys = 0.0; - mosaic[i].Xm = 0.0; - mosaic[i].flags = 0; - mosaic[i].secz = NAN; - mosaic[i].photcode = 0; - mosaic[i].skipCal = FALSE; + mosaic[i].start = startTimesMosaic[i]; + mosaic[i].stop = 0; + mosaic[i].McalPSF = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[i].McalAPER = 0.0; // note : mosaic stores only offsets relative to the original image values + mosaic[i].dMcal = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[i].dMsys = 0.0; + mosaic[i].McalChiSq = 0.0;// NAN or 0.0? + mosaic[i].flags = 0; + mosaic[i].secz = NAN; + mosaic[i].photcode = 0; + mosaic[i].skipCal = FALSE; memset (&mosaic[i].coords, 0, sizeof(Coords)); @@ -294,12 +296,15 @@ abort(); } - mosaic[j].stop = stop; - mosaic[j].Mcal = 0.0; - mosaic[j].dMcal = 0.0; - mosaic[j].Xm = 0.0; - mosaic[j].dMsys = subset[i].flags; - mosaic[j].flags = subset[i].flags; - mosaic[j].secz = subset[i].secz; - mosaic[j].photcode = GetPhotcodeEquivCodebyCode (subset[i].photcode); + mosaic[j].stop = stop; + + mosaic[j].McalPSF = 0.0; + mosaic[j].McalAPER = 0.0; + mosaic[j].dMcal = 0.0; + mosaic[j].McalChiSq = 0.0; + + mosaic[j].dMsys = subset[i].flags; + mosaic[j].flags = subset[i].flags; + mosaic[j].secz = subset[i].secz; + mosaic[j].photcode = GetPhotcodeEquivCodebyCode (subset[i].photcode); } @@ -377,14 +382,15 @@ // init the mosaic array values for (i = 0; i < Nmosaic; i++) { - mosaic[i].start = startTimesMosaic[i]; - mosaic[i].stop = 0; - mosaic[i].Mcal = 0.0; - mosaic[i].dMcal = 0.0; - mosaic[i].dMsys = 0.0; - mosaic[i].Xm = 0.0; - mosaic[i].flags = 0; - mosaic[i].secz = NAN; - mosaic[i].photcode = 0; - mosaic[i].skipCal = FALSE; + mosaic[i].start = startTimesMosaic[i]; + mosaic[i].stop = 0; + mosaic[i].McalPSF = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[i].McalAPER = 0.0; // note : mosaic stores only offsets relative to the original image values + mosaic[i].dMcal = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[i].dMsys = 0.0; + mosaic[i].McalChiSq = 0.0;// NAN or 0.0? + mosaic[i].flags = 0; + mosaic[i].secz = NAN; + mosaic[i].photcode = 0; + mosaic[i].skipCal = FALSE; memset (&mosaic[i].coords, 0, sizeof(Coords)); @@ -438,11 +444,12 @@ } mosaic[j].stop = stop; - mosaic[j].Mcal = 0.0; - mosaic[j].dMcal = 0.0; - mosaic[j].Xm = 0.0; - mosaic[j].dMsys = image[i].flags; - mosaic[j].flags = image[i].flags; - mosaic[j].secz = image[i].secz; - mosaic[j].photcode = GetPhotcodeEquivCodebyCode (image[i].photcode); + mosaic[j].McalPSF = 0.0; + mosaic[j].McalAPER = 0.0; + mosaic[j].dMcal = 0.0; + mosaic[j].McalChiSq = 0.0; + mosaic[j].dMsys = image[i].flags; + mosaic[j].flags = image[i].flags; + mosaic[j].secz = image[i].secz; + mosaic[j].photcode = GetPhotcodeEquivCodebyCode (image[i].photcode); } MARKTIME("assign images to mosaic: %f sec\n", dtime); @@ -594,7 +601,9 @@ mosaic[i].coords.cdelt2 = 1.0 / 3600.0; - mosaic[i].Mcal = 0.0; - mosaic[i].dMcal = 0.0; - mosaic[i].Xm = 0.0; + mosaic[i].McalPSF = 0.0; + mosaic[i].McalAPER = 0.0; + mosaic[i].dMcal = 0.0; + mosaic[i].dMsys = 0.0; + mosaic[i].McalChiSq = 0.0; } return; @@ -612,5 +621,5 @@ double dS, dX, dY; double R, D, Rmin, Rmax, Dmin, Dmax; - double Mcal, dMcal, Xm; + double McalPSF, McalAPER, dMcal, McalChiSq; fprintf (stderr, "*** moving Mcal from image.Mcal to mosaic.Mcal ***\n"); @@ -621,5 +630,5 @@ Dmax = Rmax = -360.0; dS = 0.0; - Mcal = dMcal = Xm = 0; + McalPSF = McalAPER = dMcal = McalChiSq = 0; for (j = 0; j < MosaicN_Image[i]; j++) { m = MosaicToImage[i][j]; @@ -678,11 +687,13 @@ // if it helps, note that ubercal uses a single zp per exposure, so the mean of those values is the same as the value - Mcal += image[m].Mcal; - dMcal += image[m].dMcal; - Xm += image[m].Xm; - - image[m].Mcal = 0.0; - image[m].dMcal = NAN; - image[m].Xm = NAN_S_SHORT; + McalPSF += image[m].McalPSF; + McalAPER += image[m].McalAPER; + dMcal += image[m].dMcal; + McalChiSq += image[m].McalChiSq; + + image[m].McalPSF = 0.0; + image[m].McalAPER = 0.0; + image[m].dMcal = NAN; + image[m].McalChiSq = NAN; } dS /= MosaicN_Image[i]; @@ -696,7 +707,8 @@ RD_to_XY (&dX, &dY, Rmax, Dmax, &mosaic[i].coords); - mosaic[i].Mcal = Mcal / MosaicN_Image[i]; - mosaic[i].dMcal = dMcal / MosaicN_Image[i]; - mosaic[i].Xm = Xm / MosaicN_Image[i]; + mosaic[i].McalPSF = McalPSF / MosaicN_Image[i]; + mosaic[i].McalAPER = McalAPER / MosaicN_Image[i]; + mosaic[i].dMcal = dMcal / MosaicN_Image[i]; + mosaic[i].McalChiSq = McalChiSq / MosaicN_Image[i]; } if (!USE_GRID) return; @@ -724,14 +736,16 @@ for (j = 0; j < MosaicN_Image[i]; j++) { im = MosaicToImage[i][j]; - image[im].Mcal += mosaic[i].Mcal; - image[im].dMcal = mosaic[i].dMcal; - image[im].Xm = mosaic[i].Xm; + image[im].McalPSF += mosaic[i].McalPSF; + image[im].McalAPER += mosaic[i].McalAPER; + image[im].dMcal = mosaic[i].dMcal; + image[im].McalChiSq = mosaic[i].McalChiSq; image[im].ubercalDist = mosaic[i].ubercalDist; - image[im].dMagSys = mosaic[i].dMsys; - image[im].nFitPhotom = mosaic[i].nFitPhotom; - image[im].flags |= (mosaic[i].flags & ID_IMAGE_PHOTOM_FEW); - image[im].flags |= (mosaic[i].flags & ID_IMAGE_PHOTOM_POOR); - } - mosaic[i].Mcal = 0.0; + image[im].dMagSys = mosaic[i].dMsys; + image[im].nFitPhotom = mosaic[i].nFitPhotom; + image[im].flags |= (mosaic[i].flags & ID_IMAGE_PHOTOM_FEW); + image[im].flags |= (mosaic[i].flags & ID_IMAGE_PHOTOM_POOR); + } + mosaic[i].McalPSF = 0.0; + mosaic[i].McalAPER = 0.0; } } @@ -886,5 +900,5 @@ if (mosaic[i].flags & IMAGE_BAD) return (NAN); - value = mosaic[i].Mcal; + value = mosaic[i].McalPSF; return (value); } @@ -911,8 +925,10 @@ off_t Nmax; int PoorImages; - double *list; - double *dlist; - double *Mlist; - double *dMlist; + double *psfMagList; + double *psfErrList; + double *brightMagList; + double *brightErrList; + double *kronMagList; + double *kronErrList; } SetMmosInfo; @@ -945,16 +961,20 @@ if (allocLists) { - ALLOCATE (info->list, double, Nmax); - ALLOCATE (info->dlist, double, Nmax); - ALLOCATE (info->Mlist, double, Nmax); - ALLOCATE (info->dMlist, double, Nmax); + ALLOCATE (info->psfMagList, double, Nmax); + ALLOCATE (info->psfErrList, double, Nmax); + ALLOCATE (info->kronMagList, double, Nmax); + ALLOCATE (info->kronErrList, double, Nmax); + ALLOCATE (info->brightMagList, double, Nmax); + ALLOCATE (info->brightErrList, double, Nmax); } } void SetMmosInfoFree (SetMmosInfo *info) { - free (info->list); - free (info->dlist); - free (info->Mlist); - free (info->dMlist); + free (info->psfMagList ); + free (info->psfErrList ); + free (info->kronMagList ); + free (info->kronErrList ); + free (info->brightMagList); + free (info->brightErrList); } @@ -1057,8 +1077,10 @@ liststats_setmode (&stats, "INNER_WTMEAN"); - double *list = info->list; - double *dlist = info->dlist; - double *Mlist = info->Mlist; - double *dMlist = info->dMlist; + double *psfMagList = info->psfMagList; + double *psfErrList = info->psfErrList; + double *kronMagList = info->kronMagList; + double *kronErrList = info->kronErrList; + double *brightMagList = info->brightMagList; + double *brightErrList = info->brightErrList; assert (Nmos >= 0); @@ -1125,23 +1147,23 @@ int N = 0; for (j = 0; j < N_onMosaic[Nmos]; j++) { - float Msys, Mrel, Mcal, Mgrid, Mflat; off_t m = MosaicToMeasure[Nmos][j]; off_t c = MosaicToCatalog[Nmos][j]; + // NOTE : we are only using Mcal == McalPSF in this function; we set McalAPER to McalPSF if (fout) { - Mcal = getMcal (m, c, flatcorr, catalog); - Mgrid = getMgrid (m, c); - Mrel = getMrel (catalog, m, c); - Mflat = getMflat (m, c, flatcorr, catalog); + float Mcal = getMcal (m, c, MAG_CLASS_PSF); + float Mgrid = getMgrid (m, c); + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + float Mflat = getMflat (m, c, flatcorr, catalog); off_t n = catalog[c].measureT[m].averef; - Msys = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); - - float delta = Msys - Mrel - Mcal - Mgrid + Mflat; + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + + float delta = MsysPSF - MrelPSF - Mcal - Mgrid + Mflat; int isBad = (catalog[c].measureT[m].dbFlags & MEAS_BAD); - fprintf (fout, "%f %f : %f %f %f %f %f : %f %d\n", catalog[c].averageT[n].R, catalog[c].averageT[n].D, Msys, Mrel, Mcal, Mgrid, Mflat, delta, isBad); + fprintf (fout, "%f %f : %f %f %f %f %f : %f %d\n", catalog[c].averageT[n].R, catalog[c].averageT[n].D, MsysPSF, MrelPSF, Mcal, Mgrid, Mflat, delta, isBad); } @@ -1150,19 +1172,27 @@ continue; } - Mcal = getMcal (m, c, flatcorr, catalog); + float Mcal = getMcal (m, c, MAG_CLASS_PSF); if (isnan(Mcal)) { info->Ncal++; continue; } - Mgrid = getMgrid (m, c); + float Mgrid = getMgrid (m, c); if (isnan(Mgrid)) { info->Ngrid ++; continue; } - Mrel = getMrel (catalog, m, c); - if (isnan(Mrel)) { + + // Mrel* is the average magnitude for this star. For PS1 stacks, we have too much + // PSF variability. We need to calibrate the PSF magnitudes separately from the + // Aperture-like magnitues. (We have an option to use the kron magnitudes or the + // other apertures here). I basically need to do this analysis separately for each + // magnitude type + + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + if (isnan(MrelPSF)) { info->Nrel ++; continue; } + float MrelKron = getMrel (catalog, m, c, MAG_CLASS_KRON, MAG_SRC_CHP); // image.Mcal is not supposed to include the flat-field correction, so we need to @@ -1171,12 +1201,13 @@ // the flat-correction. NOTE the sign of Mflat (Image.Mcal = Measure.Mcal - Mflat) - Mflat = getMflat (m, c, flatcorr, catalog); + float Mflat = getMflat (m, c, flatcorr, catalog); off_t n = catalog[c].measureT[m].averef; - Msys = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); - if (isnan(Msys)) { + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + if (isnan(MsysPSF)) { info->Nsys++; continue; } + float MsysKron = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_KRON); PhotCode *code = GetPhotcodebyCode (catalog[c].measureT[m].photcode); @@ -1193,9 +1224,11 @@ assert (Nbright >= 0); - list[N] = Msys - Mrel - Mcal - Mgrid + Mflat; - dlist[N] = MAX (catalog[c].measureT[m].dM, MIN_ERROR); + psfMagList[N] = MsysPSF - MrelPSF - Mcal - Mgrid + Mflat; + psfErrList[N] = MAX (catalog[c].measureT[m].dM, MIN_ERROR); + kronMagList[N] = MsysKron - MrelKron - Mcal - Mgrid + Mflat; + kronErrList[N] = psfErrList[N]; if (catalog[c].measureT[m].dM < IMFIT_SYS_SIGMA_LIM) { - Mlist[Nbright] = list[N]; - dMlist[Nbright] = dlist[N]; + brightMagList[Nbright] = psfMagList[N]; + brightErrList[Nbright] = psfErrList[N]; Nbright ++; } @@ -1223,11 +1256,15 @@ } - liststats (list, dlist, NULL, N, &stats); + liststats (psfMagList, psfErrList, NULL, N, &stats); if (VERBOSE2 && info->PoorImages) fprintf (stderr, "Mmos: %f %f %d %d\n", stats.mean, stats.sigma, stats.Nmeas, N); - myMosaic[0].Mcal = stats.mean; - myMosaic[0].dMcal = stats.error; + // for now, I have no reason to measure these separately for camera-level images + myMosaic[0].McalPSF = stats.mean; + myMosaic[0].dMcal = stats.error; + myMosaic[0].McalChiSq = stats.chisq; myMosaic[0].nFitPhotom = N; - myMosaic[0].Xm = 100.0*log10(stats.chisq); + + liststats (kronMagList, kronErrList, NULL, N, &stats); + myMosaic[0].McalAPER = stats.mean; if (testImage) { @@ -1237,17 +1274,17 @@ if (PLOTSTUFF) { fprintf (stderr, "Mmos: %6.3f %6.3f +/- %6.3f %5d %5d | %s\n", stats.mean, stats.median, stats.sigma, stats.Nmeas, N, image[MosaicToImage[Nmos][0]].name); - plot_setMcal (list, N, &stats, CLOUD_TOLERANCE); + plot_setMcal (psfMagList, N, &stats, CLOUD_TOLERANCE); } // bright end scatter - liststats (Mlist, dMlist, NULL, Nbright, &stats); + liststats (brightMagList, brightErrList, NULL, Nbright, &stats); myMosaic[0].dMsys = stats.sigma; - if (myMosaic[0].Mcal < -CLOUD_TOLERANCE) { - myMosaic[0].Mcal = 0.0; + if (myMosaic[0].McalPSF < -CLOUD_TOLERANCE) { + myMosaic[0].McalPSF = 0.0; } if (testImage) { - fprintf (stderr, "%f %f : %f\n", myMosaic[0].Mcal, myMosaic[0].dMsys, pow (10.0, 0.01*myMosaic[0].Xm)); + fprintf (stderr, "%f %f : %f\n", myMosaic[0].McalPSF, myMosaic[0].dMsys, myMosaic[0].McalChiSq); } @@ -1427,15 +1464,15 @@ imageOffset[i] = 0.0; - if (VERBOSE2 && (fabs(mosaic[i].Mcal) < CLOUD_TOLERANCE)) { - fprintf (stderr, "cloud-free: %s : %f\n", image[MosaicToImage[i][0]].name, mosaic[i].Mcal); + if (VERBOSE2 && (fabs(mosaic[i].McalPSF) < CLOUD_TOLERANCE)) { + fprintf (stderr, "cloud-free: %s : %f\n", image[MosaicToImage[i][0]].name, mosaic[i].McalPSF); } - if (VERBOSE2 && (mosaic[i].Mcal < -CLOUD_TOLERANCE)) { - imageOffset[i] = -mosaic[i].Mcal; - // NOTE the negative sign: down below, we are going to add in the negative of Mcal + if (VERBOSE2 && (mosaic[i].McalPSF < -CLOUD_TOLERANCE)) { + imageOffset[i] = -mosaic[i].McalPSF; + // NOTE the negative sign: down below, we are going to add in the negative of McalPSF // to this image, and the propagated mean values for other images - fprintf (stderr, "anti-clouds: %s : %f\n", image[MosaicToImage[i][0]].name, mosaic[i].Mcal); + fprintf (stderr, "anti-clouds: %s : %f\n", image[MosaicToImage[i][0]].name, mosaic[i].McalPSF); } - if (VERBOSE2 && (mosaic[i].Mcal > CLOUD_TOLERANCE)) { - fprintf (stderr, "cloudy : %s : %f\n", image[MosaicToImage[i][0]].name, mosaic[i].Mcal); + if (VERBOSE2 && (mosaic[i].McalPSF > CLOUD_TOLERANCE)) { + fprintf (stderr, "cloudy : %s : %f\n", image[MosaicToImage[i][0]].name, mosaic[i].McalPSF); } } @@ -1540,5 +1577,5 @@ } - // a negative cloud image (cloud: Mcal > 0; anti-clouds: Mcal < 0; imageOffset = -Mcal) + // a negative cloud image (cloud: McalPSF > 0; anti-clouds: McalPSF < 0; imageOffset = -McalPSF) if (imageOffset[i] > 0.0) continue; @@ -1564,5 +1601,5 @@ // find the images / mosaics with negative clouds and save their offset for (i = 0; i < Nmosaic; i++) { - mosaic[i].Mcal += imageOffset[i]; + mosaic[i].McalPSF += imageOffset[i]; } @@ -1605,5 +1642,5 @@ if (KEEP_UBERCAL && (mosaic[i].flags & ID_IMAGE_PHOTOM_UBERCAL)) continue; - list[n] = mosaic[i].Mcal; + list[n] = mosaic[i].McalPSF; dlist[n] = 1; n++; @@ -1677,9 +1714,9 @@ c = MosaicToCatalog[i][j]; - Mcal = getMcal (m, c, flatcorr, catalog); + Mcal = getMcal (m, c, MAG_CLASS_PSF); if (isnan(Mcal)) continue; Mgrid = getMgrid (m, c); if (isnan(Mgrid)) continue; - Mrel = getMrel (catalog, m, c); + Mrel = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); if (isnan(Mrel)) continue; N++; @@ -1719,5 +1756,5 @@ if (KEEP_UBERCAL && (mosaic[i].flags & ID_IMAGE_PHOTOM_UBERCAL)) continue; - list[n] = pow (10.0, 0.01*mosaic[i].Xm); + list[n] = mosaic[i].McalChiSq; dlist[n] = 1; n++; @@ -1751,5 +1788,5 @@ if (mosaic[i].flags & IMAGE_BAD) continue; if (mosaic[i].skipCal) continue; - mlist[N] = mosaic[i].Mcal; + mlist[N] = mosaic[i].McalPSF; slist[N] = mosaic[i].dMcal; dlist[N] = 1; @@ -1781,5 +1818,5 @@ Nscatter ++; } - if (fabs(mosaic[i].Mcal - MedOffset) > MaxOffset) { + if (fabs(mosaic[i].McalPSF - MedOffset) > MaxOffset) { mark = TRUE; Noffset ++; @@ -1858,5 +1895,5 @@ for (i = 0; i < Nmosaic; i++) { - Mlist[i] = mosaic[i].Mcal; + Mlist[i] = mosaic[i].McalPSF; dlist[i] = mosaic[i].dMcal; xlist[i] = mosaic[i].secz; Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/StarOps.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/StarOps.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/StarOps.c (revision 40477) @@ -34,22 +34,66 @@ } -float getMrel (Catalog *catalog, off_t meas, int cat) { - - int Nsec, Nsecfilt, photcode; - int ave; - float value; - - ave = catalog[cat].measureT[meas].averef; - photcode = catalog[cat].measureT[meas].photcode; +// +float getMrel (Catalog *catalog, off_t meas, int cat, dvoMagClassType class, dvoMagSourceType source) { + + int ave = catalog[cat].measureT[meas].averef; + int photcode = catalog[cat].measureT[meas].photcode; int ecode = GetPhotcodeEquivCodebyCode (photcode); - Nsec = GetPhotcodeNsec(ecode); - Nsecfilt = GetPhotcodeNsecfilt (); + int Nsec = GetPhotcodeNsec(ecode); + int Nsecfilt = GetPhotcodeNsecfilt (); + + int entry = Nsecfilt*ave+Nsec; + + SecFilt *secfilt = &catalog[cat].secfilt[entry]; // is this star OK? - if (catalog[cat].secfilt[Nsecfilt*ave+Nsec].flags & STAR_BAD) return (NAN); - - value = catalog[cat].secfilt[Nsecfilt*ave+Nsec].M; - return (value); + if (secfilt->flags & STAR_BAD) return (NAN); + + switch (class) { + case MAG_CLASS_PSF: + switch (source) { + case MAG_SRC_CHP: + return secfilt->MpsfChp; + case MAG_SRC_WRP: + return secfilt->MpsfWrp; + case MAG_SRC_STK: + return secfilt->MpsfStk; + default: + return NAN; + } + break; + case MAG_CLASS_KRON: + switch (source) { + case MAG_SRC_CHP: + return secfilt->MkronChp; + case MAG_SRC_WRP: + return secfilt->MkronWrp; + case MAG_SRC_STK: + return secfilt->MkronStk; + default: + return NAN; + } + break; + case MAG_CLASS_APER: + switch (source) { + case MAG_SRC_CHP: + return secfilt->MapChp; + case MAG_SRC_WRP: + return secfilt->MapWrp; + case MAG_SRC_STK: + return secfilt->MapStk; + default: + return NAN; + } + break; + case MAG_CLASS_NONE: + case MAG_CLASS_DEV: // DeVaucouleur Model (only for galphot) + case MAG_CLASS_EXP: // Exponential Model (only for galphot) + case MAG_CLASS_SER: // Sersic Model (only for galphot) + return NAN; + break; + } + return NAN; // should not be able to reach here } @@ -429,5 +473,4 @@ int i; off_t j, k, m; - float Mcal, Mmos, Mgrid; MEAS_BAD = ID_MEAS_NOCAL; @@ -440,10 +483,18 @@ for (k = 0; k < catalog[i].averageT[j].Nmeasure; k++, m++) { if (catalog[i].measureT[m].dbFlags & MEAS_BAD) continue; - Mcal = getMcal (m, i, flatcorr, catalog); - if (isnan(Mcal)) continue; - Mmos = getMmos (m, i); + float McalPSF = getMcal (m, i, MAG_CLASS_PSF); + float McalAPER = getMcal (m, i, MAG_CLASS_APER); + if (isnan(McalPSF)) continue; + float Mmos = getMmos (m, i); if (isnan(Mmos)) continue; - Mgrid = getMgrid (m, i); + float Mgrid = getMgrid (m, i); if (isnan(Mgrid)) continue; + + // Note that this operation is setting measure->McalAPER to image->McalAPER only + // for the stacks. Elsewhere (setMrelCatalog) we are using image->McalPSF for all + // types of measurements EXCEPT stacks. + + // we need to use McalAPER for stacks (and only stacks) + int useStackAper = !USE_MCAL_PSF_FOR_STACK_APER && isGPC1stack(catalog[i].measureT[m].photcode); // note that measurements for which the image is not selected will not be modified @@ -451,8 +502,9 @@ // set the output calibration - catalog[i].measure[m].Mcal = Mcal + Mmos + Mgrid; + catalog[i].measure[m].McalPSF = McalPSF + Mmos + Mgrid; + catalog[i].measure[m].McalAPER = useStackAper ? McalAPER + Mmos + Mgrid : McalPSF + Mmos + Mgrid; if (catalog[i].measureT[m].dbFlags & ID_MEAS_PHOTOM_UBERCAL) { - myAssert (isfinite(catalog[i].measure[m].Mcal), "oops, broke an ubercal mag"); + myAssert (isfinite(catalog[i].measure[m].McalPSF), "oops, broke an ubercal mag"); } } @@ -501,4 +553,26 @@ } +int dumpMags (FILE *fout, Catalog *catalog, int Ncatalog) { + + int i, n; + off_t j; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + for (i = 0; i < Ncatalog; i++) { + for (j = 0; (j < catalog[i].Naverage) && (j < 2); j++) { + fprintf (fout, "%08x %08x %10.6f %10.6f : \n", catalog[i].averageT[j].catID, catalog[i].averageT[j].objID, catalog[i].averageT[j].R, catalog[i].averageT[j].D); + for (n = 0; n < 5; n++) { + fprintf (fout, "secf %5d | %6.3f %6.3f | %6.3f %6.3f | %6.3f %6.3f\n", n, catalog[i].secfilt[j*Nsecfilt + n].MpsfChp, catalog[i].secfilt[j*Nsecfilt + n].MkronChp, catalog[i].secfilt[j*Nsecfilt + n].MpsfStk, catalog[i].secfilt[j*Nsecfilt + n].MkronStk, catalog[i].secfilt[j*Nsecfilt + n].MpsfWrp, catalog[i].secfilt[j*Nsecfilt + n].MkronWrp); + } + int m = catalog[i].averageT[j].measureOffset; + for (n = 0; n < catalog[i].averageT[j].Nmeasure; n++) { + fprintf (fout, "meas %5d %5d | %6.3f %6.3f | %6.3f %6.3f\n", m+n, catalog[i].measureT[m+n].photcode, catalog[i].measureT[m+n].M, catalog[i].measureT[m+n].Mkron, catalog[i].measureT[m+n].McalPSF, catalog[i].measureT[m+n].McalAPER); + } + } + } + return TRUE; +} + void clean_stars (Catalog *catalog, int Ncatalog) { @@ -513,5 +587,5 @@ if (VERBOSE) fprintf (stderr, "marking poor stars\n"); - /* find Xm median -> ChiSq lim must be > median */ + /* find Mchisq median -> ChiSq lim must be > median */ for (i = Ntot = 0; i < Ncatalog; i++) { Ntot += catalog[i].Naverage; @@ -535,5 +609,5 @@ if (isnan(Mchisq)) continue; xlist[Ntot] = Mchisq; - slist[Ntot] = catalog[i].secfilt[Nsecfilt*j+Nsec].dM; + slist[Ntot] = catalog[i].secfilt[Nsecfilt*j+Nsec].dMpsfChp; dlist[Ntot] = 1; Ntot ++; @@ -551,5 +625,5 @@ for (i = 0; i < Ncatalog; i++) { for (j = 0; j < catalog[i].Naverage; j++) { - dM = catalog[i].secfilt[Nsecfilt*j+Nsec].dM; + dM = catalog[i].secfilt[Nsecfilt*j+Nsec].dMpsfChp; float Mchisq = catalog[i].secfilt[Nsecfilt*j+Nsec].Mchisq; mark = (dM > MaxScatter) || (isnan(Mchisq)) || (Mchisq > MaxChisq); @@ -578,4 +652,6 @@ // NSIGMA_REJECT (5) sigma of the mean +// this function only operations on the PSF magnitudes + # define NSIGMA_CLIP 3.0 # define NSIGMA_REJECT 5.0 @@ -644,5 +720,5 @@ // NOTE: we do not skip MEAS_BAD because this measurement is just an internal assessment of the outliers - Mcal = getMcal (m, i, flatcorr, catalog); + Mcal = getMcal (m, i, MAG_CLASS_PSF); if (isnan(Mcal)) { Ncal ++; continue; } Mmos = getMmos (m, i); @@ -695,5 +771,5 @@ // NOTE: we do not skip MEAS_BAD because this measurement is just an internal assessment of the outliers - Mcal = getMcal (m, i, flatcorr, catalog); + Mcal = getMcal (m, i, MAG_CLASS_PSF); if (isnan(Mcal)) continue; Mmos = getMmos (m, i); @@ -776,5 +852,5 @@ int ecode = GetPhotcodeEquivCodebyCode (catalog[i].measureT[m].photcode); if (ecode != seccode) { continue;} - Mcal = getMcal (m, i, flatcorr, catalog); + Mcal = getMcal (m, i, MAG_CLASS_PSF); if (isnan(Mcal)) { continue;} Mmos = getMmos (m, i); @@ -865,5 +941,5 @@ if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) continue; - dM = catalog[i].secfilt[Nsecfilt*j+Nsec].dM; + dM = catalog[i].secfilt[Nsecfilt*j+Nsec].dMpsfChp; if (isnan(dM)) continue; list[n] = dM; @@ -906,5 +982,5 @@ for (j = 0; j < catalog[i].Naverage; j++) { if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) continue; - dMrel = catalog[i].secfilt[Nsecfilt*j+Nsec].dM; + dMrel = catalog[i].secfilt[Nsecfilt*j+Nsec].dMpsfChp; bin = dMrel / 0.00025; bin = MAX (0, MIN (NBIN-1, bin)); @@ -946,5 +1022,5 @@ for (j = 0; j < catalog[i].Naverage; j++) { if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) continue; - xlist[N] = catalog[i].secfilt[Nsecfilt*j+Nsec].M; + xlist[N] = catalog[i].secfilt[Nsecfilt*j+Nsec].MpsfChp; value = catalog[i].secfilt[Nsecfilt*j+Nsec].Mchisq; if (isnan((double)(value))) continue; Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/args.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/args.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/args.c (revision 40477) @@ -136,4 +136,10 @@ remove_argument (N, &argc, argv); } + SKIP_PARALLEL_GROUPS = 0; + if ((N = get_argument (argc, argv, "-skip-parallel-groups"))) { + remove_argument (N, &argc, argv); + SKIP_PARALLEL_GROUPS = atoi(argv[N]); + remove_argument (N, &argc, argv); + } // elements needed for parallel regions / parallel images @@ -325,4 +331,9 @@ remove_argument (N, &argc, argv); CALIBRATE_STACKS_AND_WARPS = TRUE; + } + USE_MCAL_PSF_FOR_STACK_APER = FALSE; + if ((N = get_argument (argc, argv, "-use-mcal-psf-for-stack-aper"))) { + remove_argument (N, &argc, argv); + USE_MCAL_PSF_FOR_STACK_APER = TRUE; } @@ -544,4 +555,5 @@ PARALLEL_MANUAL = FALSE; PARALLEL_SERIAL = FALSE; + SKIP_PARALLEL_GROUPS = 0; HOST_ID = 0; @@ -694,4 +706,10 @@ remove_argument (N, &argc, argv); KEEP_UBERCAL = FALSE; + } + + USE_MCAL_PSF_FOR_STACK_APER = FALSE; + if ((N = get_argument (argc, argv, "-use-mcal-psf-for-stack-aper"))) { + remove_argument (N, &argc, argv); + USE_MCAL_PSF_FOR_STACK_APER = TRUE; } Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/bcatalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/bcatalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/bcatalog.c (revision 40477) @@ -153,10 +153,12 @@ if (subcatalog[0].measureT[Nmeasure].dbFlags & ID_MEAS_PHOTOM_UBERCAL) { if (!KEEP_UBERCAL) { - subcatalog[0].measureT[Nmeasure].Mcal = 0.0; + subcatalog[0].measureT[Nmeasure].McalPSF = 0.0; + subcatalog[0].measureT[Nmeasure].McalAPER = 0.0; subcatalog[0].measureT[Nmeasure].dbFlags &= ~ID_MEAS_PHOTOM_UBERCAL; } } else { if (getImageEntry (Nmeasure, Ncat) >= 0) { - subcatalog[0].measureT[Nmeasure].Mcal = 0.0; + subcatalog[0].measureT[Nmeasure].McalPSF = 0.0; + subcatalog[0].measureT[Nmeasure].McalAPER = 0.0; } } Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/launch_region_hosts.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/launch_region_hosts.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/launch_region_hosts.c (revision 40477) @@ -58,22 +58,25 @@ strextend (&command, "-cloud-limit %f", CLOUD_TOLERANCE); - if (VERBOSE) strextend (&command, "-v"); - if (VERBOSE2) strextend (&command, "-vv"); - if (RESET) strextend (&command, "-reset"); - if (RESET_ZEROPTS) strextend (&command, "-reset-zpts"); - if (!KEEP_UBERCAL) strextend (&command, "-reset-ubercal"); - if (DophotSelect) strextend (&command, "-dophot %d", DophotValue); - if (ImagSelect) strextend (&command, "-instmag %f %f", ImagMin, ImagMax); - if (MaxDensityUse) strextend (&command, "-max-density %f", MaxDensityValue); - if (SyntheticPhotometry) strextend (&command, "-synthphot"); - if (USE_BASIC_CHECK) strextend (&command, "-basic-image-search"); + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (RESET) strextend (&command, "-reset"); + if (RESET_ZEROPTS) strextend (&command, "-reset-zpts"); + if (!KEEP_UBERCAL) strextend (&command, "-reset-ubercal"); + if (DophotSelect) strextend (&command, "-dophot %d", DophotValue); + if (ImagSelect) strextend (&command, "-instmag %f %f", ImagMin, ImagMax); + if (MaxDensityUse) strextend (&command, "-max-density %f", MaxDensityValue); + if (SyntheticPhotometry) strextend (&command, "-synthphot"); + if (USE_BASIC_CHECK) strextend (&command, "-basic-image-search"); - if (UPDATE) strextend (&command, "-update"); - if (MOSAIC_ZEROPT) strextend (&command, "-mosaic"); - if (FREEZE_IMAGES) strextend (&command, "-imfreeze"); - if (FREEZE_MOSAICS) strextend (&command, "-mosfreeze"); - if (PARALLEL) strextend (&command, "-parallel"); - if (PARALLEL_MANUAL) strextend (&command, "-parallel-manual"); - if (PARALLEL_SERIAL) strextend (&command, "-parallel-serial"); + if (UPDATE) strextend (&command, "-update"); + if (MOSAIC_ZEROPT) strextend (&command, "-mosaic"); + if (FREEZE_IMAGES) strextend (&command, "-imfreeze"); + if (FREEZE_MOSAICS) strextend (&command, "-mosfreeze"); + if (CALIBRATE_STACKS_AND_WARPS) strextend (&command, "-only-stacks-and-warps"); + if (USE_MCAL_PSF_FOR_STACK_APER) { strextend (&command, "-use-mcal-psf-for-stack-aper"); } + + if (PARALLEL) strextend (&command, "-parallel"); + if (PARALLEL_MANUAL) strextend (&command, "-parallel-manual"); + if (PARALLEL_SERIAL) strextend (&command, "-parallel-serial"); // XXX deprecate this if we are happy with the new version Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/liststats.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/liststats.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/liststats.c (revision 40477) @@ -194,5 +194,5 @@ # define WEIGHT_THRESHOLD 0.3 -int fit_least_squares (double *fit, double *y, double *dy, double *wgt, double *wt, int Npts); +int fit_least_squares (double *fit, double *err, double *y, double *dy, double *wgt, double *wt, int Npts); double VectorFractionInterpolate (double *values, float fraction, int Npts); double weight_cauchy (double x); @@ -253,5 +253,5 @@ double oldValue = value; - if (!fit_least_squares (&value, dataset->flxlist, dataset->errlist, dataset->wgtlist, dataset->wtvals, Npoints)) { + if (!fit_least_squares (&value, NULL, dataset->flxlist, dataset->errlist, dataset->wgtlist, dataset->wtvals, Npoints)) { value = oldValue; break; @@ -330,5 +330,5 @@ } - if (!fit_least_squares (&value, dataset->ysample, dataset->dysample, dataset->wtsample, NULL, Nkeep)) continue; + if (!fit_least_squares (&value, NULL, dataset->ysample, dataset->dysample, dataset->wtsample, NULL, Nkeep)) continue; dataset->bvalue[Nboot] = value; @@ -342,9 +342,16 @@ stats->error = (Shi - Slo) / 2.0; + // bootstrap can sometimes yield an excessively-optimistic result for the error. Do not let + // the reported error be smaller than the formal error + double errvalue; + if (fit_least_squares (&value, &errvalue, dataset->ykeep, dataset->dykeep, dataset->wtkeep, NULL, Nkeep)) { + stats->error = MAX (stats->error, errvalue); + } + return TRUE; } // wgt is externally-supplied weight, wt is optional -int fit_least_squares (double *fit, double *y, double *dy, double *wgt, double *wt, int Npts) { +int fit_least_squares (double *fit, double *err, double *y, double *dy, double *wgt, double *wt, int Npts) { int i; @@ -364,5 +371,6 @@ } if (S0 == 0.0) return FALSE; - *fit = S1 / S0; + *fit = S1 / S0; + if (err) { *err = 1.0 / S0; } return TRUE; } Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/load_images.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/load_images.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/load_images.c (revision 40477) @@ -52,4 +52,24 @@ free (subset); subset = image; + } + } + + // reset image values as needed. always allow 'few' images to succeed, if possible (new + // images / detections may have been added + for (off_t i = 0; i < Nsubset; i++) { + if (subset[i].flags & ID_IMAGE_PHOTOM_FEW) { + subset[i].flags &= ~(ID_IMAGE_PHOTOM_FEW | ID_IMAGE_PHOTOM_POOR); + } + if (RESET) { + if (RESET_ZEROPTS) { + if (!KEEP_UBERCAL || !(subset[i].flags & ID_IMAGE_PHOTOM_UBERCAL)) { + subset[i].McalPSF = 0.0; + subset[i].McalAPER = 0.0; + subset[i].dMcal = NAN; + subset[i].flags &= ~ID_IMAGE_PHOTOM_UBERCAL; + } + } + subset[i].flags &= ~ID_IMAGE_PHOTOM_POOR; + subset[i].ubercalDist = 1000; } } Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/plot_scatter.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/plot_scatter.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/plot_scatter.c (revision 40477) @@ -40,5 +40,5 @@ if (catalog[i].measureT[m].dbFlags & MEAS_BAD) continue; - Mcal = getMcal (m, i, flatcorr, catalog); + Mcal = getMcal (m, i, MAG_CLASS_PSF); if (isnan(Mcal)) continue; Mmos = getMmos (m, i); @@ -47,5 +47,5 @@ if (isnan(Mgrid)) continue; - Mrel = catalog[i].secfilt[Nsecfilt*j+Nsec].M; + Mrel = catalog[i].secfilt[Nsecfilt*j+Nsec].MpsfChp; if (isnan(Mrel)) continue; Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/plotstuff.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/plotstuff.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/plotstuff.c (revision 40477) @@ -179,5 +179,5 @@ KapaInitGraph (graphdata); - graphdata[0].style = 2; + graphdata[0].style = KAPA_PLOT_POINTS; /* points */ graphdata[0].ptype = 2; graphdata[0].ltype = 0; Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/reload_catalogs.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/reload_catalogs.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/reload_catalogs.c (revision 40477) @@ -180,4 +180,5 @@ for (i = 0; i < Ngroups; i++) { // update only a group of unique machines at a time + if (i < SKIP_PARALLEL_GROUPS) continue; reload_catalog_parallel_group (&groups[i], sky, imageFile); } @@ -247,4 +248,5 @@ if (SYNTH_ZERO_POINTS) { strextend (&command, "-synthphot-zpts %s", SYNTH_ZERO_POINTS); } if (USE_BASIC_CHECK) { strextend (&command, "-basic-image-search"); } + if (USE_MCAL_PSF_FOR_STACK_APER) { strextend (&command, "-use-mcal-psf-for-stack-aper"); } if (!(STAGES & STAGE_CHIP)) { strextend (&command, "-skip-chip"); } Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_images.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_images.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_images.c (revision 40477) @@ -42,8 +42,11 @@ MARKTIME("-- load images: %f sec\n", dtime); + // checkImages("load images"); + /* unlock, if we can (else, unlocked below) */ if (!UPDATE) dvo_image_unlock (&db); // load the flat correction table (if defined) + // we no longer need to do this: the flatcorrection is now saved in measure.Mflat # if (0) char flatcorrfile[256]; @@ -60,4 +63,6 @@ /* add in a loop over the catalogs calling dvo_catalog_chipcoords */ + // checkImages("load catalog"); + /* match measurements with images, mosaics */ initImageBins (catalog, Ncatalog, TRUE); @@ -77,4 +82,5 @@ setExclusions (catalog, Ncatalog, TRUE); + // checkImages("set Exclu"); global_stats (catalog, Ncatalog, flatcorr, 0); @@ -108,9 +114,18 @@ plot_scatter (catalog, Ncatalog, flatcorr); } + if (VERBOSE2) dumpMags (stderr, catalog, Ncatalog); + // checkImages("start loop"); + setMrel (catalog, Ncatalog, flatcorr); // threaded + if (VERBOSE2) dumpMags (stderr, catalog, Ncatalog); + // checkImages("set Mrel"); + if (PLOTSTUFF) { plot_scatter (catalog, Ncatalog, flatcorr); } setMcal (catalog, FALSE, flatcorr); + // checkImages("set Mcal"); + if (VERBOSE2) dumpMags (stderr, catalog, Ncatalog); + setMmos (catalog, FALSE, flatcorr); setMgrid (catalog, flatcorr); @@ -213,4 +228,6 @@ // ensure the db format is updated dbX.format = dvo_catalog_catformat (UPDATE_CATFORMAT); + gfits_modify (&dbX.header, "FORMAT", "%s", 1, UPDATE_CATFORMAT); + char photcodeFile[1024]; sprintf (photcodeFile, "%s/Photcodes.dat", CATDIR); @@ -257,4 +274,5 @@ // ensure the db format is updated db.format = dvo_catalog_catformat (UPDATE_CATFORMAT); + gfits_modify (&db.header, "FORMAT", "%s", 1, UPDATE_CATFORMAT); } if (CALIBRATE_STACKS_AND_WARPS || (NLOOP > 0)) { Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_objects.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_objects.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_objects.c (revision 40477) @@ -251,4 +251,5 @@ if (USE_BASIC_CHECK) { strextend (&command, "-basic-image-search"); } if (USE_ALL_IMAGES) { strextend (&command, "-use-all-images"); } + if (USE_MCAL_PSF_FOR_STACK_APER) { strextend (&command, "-use-mcal-psf-for-stack-aper"); } if (!(STAGES & STAGE_CHIP)) { strextend (&command, "-skip-chip"); } Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_parallel_images.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_parallel_images.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_parallel_images.c (revision 40477) @@ -36,7 +36,4 @@ client_logger_message ("loaded images\n"); - // once we have read this table, we should remove it for repeat runs - // unlink (IMAGE_TABLE); // XXX a bit risky, add some protection? - makeMosaics (image, Nimage, TRUE); @@ -44,4 +41,5 @@ // load the flat correction table (if defined) + // we no longer need to do this: the flatcorrection is now saved in measure.Mflat # if (0) char flatcorrfile[256]; @@ -150,7 +148,8 @@ client_logger_message ("all hosts are done the loops\n"); - /* set Mcal & Mmos for bad images */ - setMcal (catalog, TRUE, flatcorr); - setMmos (catalog, TRUE, flatcorr); + /* set Mcal & Mmos for bad images (for stack_and_warps, force images to be measured) */ + int onlyPoorImages = !CALIBRATE_STACKS_AND_WARPS; + setMcal (catalog, onlyPoorImages, flatcorr); + setMmos (catalog, onlyPoorImages, flatcorr); MARKTIME("-- finalize Mcal values: %f sec\n", dtime); @@ -171,4 +170,5 @@ freeGridBins (Ncatalog); freeImages((char *)image); + free (image); SkyListFree(skylist); Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_parallel_regions.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_parallel_regions.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_parallel_regions.c (revision 40477) @@ -51,4 +51,5 @@ // ensure the db format is updated db.format = dvo_catalog_catformat (UPDATE_CATFORMAT); + gfits_modify (&db.header, "FORMAT", "%s", 1, UPDATE_CATFORMAT); } @@ -84,5 +85,9 @@ // save the changes to the image parameters - dvo_image_save (&db, VERBOSE); // this function modifies the db.ftable.buffer: do not free stored Image table + dvo_image_save (&db, VERBOSE); + + // dvo_image_save frees db.ftable.buffer (== image) and replaces it: do not free stored Image table + clearImages (); + dvo_image_unlock (&db); MARKTIME ("finished relphot -parallel-regions: %f sec total\n", dtime); Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_synthphot_catalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_synthphot_catalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/relphot_synthphot_catalog.c (revision 40477) @@ -92,8 +92,8 @@ float *value = (float *) zpts->matrix[Nsec].buffer; float ZP = !isnan(value[Npix]) ? value[Npix] : 0.0; - secfilt[Nsec].M = measure[measSYN[i]].M + ZP; - secfilt[Nsec].dM = 0.6; - secfilt[Nsec].Mchisq = NAN; - secfilt[Nsec].flags |= ID_SECF_USE_SYNTH; + secfilt[Nsec].MpsfChp = measure[measSYN[i]].M + ZP; + secfilt[Nsec].dMpsfChp = 0.6; + secfilt[Nsec].Mchisq = NAN; + secfilt[Nsec].flags |= ID_SECF_USE_SYNTH; } } Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/select_images.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/select_images.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/select_images.c (revision 40477) @@ -251,20 +251,4 @@ image[nimage] = timage[i]; inSubset[i] = TRUE; - /* always allow 'few' images to succeed, if possible (new images / detections may have - * been added) */ - if (image[nimage].flags & ID_IMAGE_PHOTOM_FEW) { - image[nimage].flags &= ~(ID_IMAGE_PHOTOM_FEW | ID_IMAGE_PHOTOM_POOR); - } - if (RESET) { - if (RESET_ZEROPTS) { - if (!KEEP_UBERCAL || !(image[nimage].flags & ID_IMAGE_PHOTOM_UBERCAL)) { - image[nimage].Mcal = 0.0; - image[nimage].dMcal = NAN; - image[nimage].flags &= ~ID_IMAGE_PHOTOM_UBERCAL; - } - } - image[nimage].flags &= ~ID_IMAGE_PHOTOM_POOR; - image[nimage].ubercalDist = 1000; - } line_number[nimage] = i; nimage ++; Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/setMrelCatalog.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/setMrelCatalog.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/setMrelCatalog.c (revision 40477) @@ -2,6 +2,6 @@ # if (0) -# define TEST_OBJ_ID 0x0000000e -# define TEST_CAT_ID 0x000076ee +# define TEST_OBJ_ID 0x000149b0 +# define TEST_CAT_ID 0x00000001 # else # define TEST_OBJ_ID 0 @@ -102,4 +102,8 @@ // 3) UBERCAL measurements can have their weight increased by a big factor to help tie down the averages // 4) some reference photcode of some kind can be specified as fixed and have a high weight + +// Although I calculate McalAPER for exposures, I am only using McalPSF for chips. Note +// in StarOps.c:setMcalOutput I am setting measure->McalAPER to image->McalPSF for chips +// and warps, but not stacks // set mean of chip measurements (selected by photcode range for now): @@ -224,9 +228,8 @@ // overlaps). Msys + measure.Mcal is our best guess of the true magnitude Mmos = Mgrid = 0; - Mcal = measureT[k].Mcal; // check that this is zero for loaded REF value + Mcal = measureT[k].McalPSF; // check that this is zero for loaded REF value } else { - // getMcal_alt returns image[].Mcal modified by flatcorr(image.photom_map_id,x,y) - // NOTE: getMcal_alt does not include measure.Mflat - Mcal = getMcal_alt (meas, cat, flatcorr, measureT[k].Xccd, measureT[k].Yccd); + // getMcal returns image[].Mcal; note the flat-field correction is stored in measure.Mflat + Mcal = getMcal (meas, cat, MAG_CLASS_PSF); if (isnan(Mcal)) SKIP_THIS_MEAS(Ncal); Mmos = getMmos (meas, cat); @@ -268,18 +271,21 @@ } - float Mkron, dMkron; + // NOTE: we need to calculate the averge chip Kron on each pass to be able to calibrate the stacks + float Mkron = PhotCatTiny (&measureT[k], MAG_CLASS_KRON); + float dMkron; if (isSetMrelFinal) { - Mkron = PhotCat (&measure[k], MAG_CLASS_KRON); dMkron = MAX (hypot(measure[k].dMkron, code->photomErrSys), MIN_ERROR); - if (CHECK_VALID_MAG(Mkron, dMkron)) { - int Nkron = results->kronData[Nsec].Nlist; - results->kronData[Nsec].flxlist[Nkron] = Mkron - Mflat - Mcal - Mmos - Mgrid; // this is consistent with PhotRel - results->kronData[Nsec].errlist[Nkron] = dMkron; - results->kronData[Nsec].wgtlist[Nkron] = isUbercal ? UBERCAL_WEIGHT : 1.0; - results->kronData[Nsec].ranking[Nkron] = measureRank[k]; - results->kronData[Nsec].measSeq[Nkron] = k; - results->kronData[Nsec].msklist[Nkron] = 0; - results->kronData[Nsec].Nlist ++; - } + } else { + dMkron = MAX (hypot(measureT[k].dM, code->photomErrSys), MIN_ERROR); + } + if (CHECK_VALID_MAG(Mkron, dMkron)) { + int Nkron = results->kronData[Nsec].Nlist; + results->kronData[Nsec].flxlist[Nkron] = Mkron - Mflat - Mcal - Mmos - Mgrid; // this is consistent with PhotRel + results->kronData[Nsec].errlist[Nkron] = dMkron; + results->kronData[Nsec].wgtlist[Nkron] = isUbercal ? UBERCAL_WEIGHT : 1.0; + results->kronData[Nsec].ranking[Nkron] = measureRank ? measureRank[k] : 0; + results->kronData[Nsec].measSeq[Nkron] = k; + results->kronData[Nsec].msklist[Nkron] = 0; + results->kronData[Nsec].Nlist ++; } @@ -377,8 +383,8 @@ } } - secfilt[Nsec].M = Mpsf + ZP; - secfilt[Nsec].dM = 0.6; - secfilt[Nsec].Mchisq = 0.0; - secfilt[Nsec].flags |= ID_SECF_USE_SYNTH; + secfilt[Nsec].MpsfChp = Mpsf + ZP; + secfilt[Nsec].dMpsfChp = 0.6; + secfilt[Nsec].Mchisq = 0.0; + secfilt[Nsec].flags |= ID_SECF_USE_SYNTH; continue; } @@ -386,13 +392,12 @@ // if too few valid measurements meet the minimum criteria, go to the next entry StatType *psfstats = &results->psfstats; - int Nranking = MAG_STATS_BY_RANKING (&results->psfData[Nsec], psfstats); - if (Nranking < Nminmeas) { + int NrankingPSF = MAG_STATS_BY_RANKING (&results->psfData[Nsec], psfstats); + if (NrankingPSF < Nminmeas) { secfilt[Nsec].flags |= ID_OBJ_FEW; } else { - secfilt[Nsec].M = psfstats->mean; - secfilt[Nsec].dM = psfstats->error; - secfilt[Nsec].Mchisq = (psfstats->Nmeas > 1) ? psfstats->chisq : NAN; - } - int minRankPSF = (Nranking > 0) ? results->psfData[Nsec].ranking[0] : 10; + secfilt[Nsec].MpsfChp = psfstats->mean; + secfilt[Nsec].dMpsfChp = psfstats->error; + secfilt[Nsec].Mchisq = (psfstats->Nmeas > 1) ? psfstats->chisq : NAN; + } // when running -averages, we have no information about the images, so we cannot set this @@ -401,4 +406,13 @@ } + StatType *kronstats = &results->kronstats; + int NrankingKRON = MAG_STATS_BY_RANKING (&results->kronData[Nsec], kronstats); + if (NrankingKRON) { + secfilt[Nsec].MkronChp = kronstats->mean; + secfilt[Nsec].dMkronChp = kronstats->error; + secfilt[Nsec].sMkronChp = kronstats->sigma; + secfilt[Nsec].NusedKron = NrankingKRON; + } + if (isSetMrelFinal) { if ((average[0].objID == TEST_OBJ_ID) && (average[0].catID == TEST_CAT_ID)) { @@ -406,12 +420,9 @@ } - // mark the measurements matching this ranking - markMeasureByRanking (&results->psfData[Nsec], measure, minRankPSF, ID_MEAS_PHOTOM_PSF, ID_MEAS_MASKED_PSF); - - if (Nranking) { - secfilt[Nsec].Mstdev = psfstats->sigma; - secfilt[Nsec].Nused = psfstats->Nmeas; - secfilt[Nsec].Mmax = psfstats->max; - secfilt[Nsec].Mmin = psfstats->min; + if (NrankingPSF) { + secfilt[Nsec].sMpsfChp = psfstats->sigma; + secfilt[Nsec].Nused = psfstats->Nmeas; + secfilt[Nsec].Mmax = psfstats->max; + secfilt[Nsec].Mmin = psfstats->min; } secfilt[Nsec].psfQfMax = results->psfQfMax[Nsec]; @@ -421,25 +432,21 @@ MAX_NOT_NAN (psfQfPerfMax, secfilt[Nsec].psfQfPerfMax); + // mark the measurements matching this ranking + int minRankPSF = (NrankingPSF > 0) ? results->psfData[Nsec].ranking[0] : 10; + markMeasureByRanking (&results->psfData[Nsec], measure, minRankPSF, ID_MEAS_PHOTOM_PSF, ID_MEAS_MASKED_PSF); + + int minRankKron = (NrankingKRON > 0) ? results->kronData[Nsec].ranking[0] : 10; + markMeasureByRanking (&results->kronData[Nsec], measure, minRankKron, ID_MEAS_PHOTOM_KRON, ID_MEAS_MASKED_KRON); + StatType *apstats = &results->apstats; - Nranking = MAG_STATS_BY_RANKING (&results->aperData[Nsec], apstats); - if (Nranking) { - secfilt[Nsec].Map = apstats->mean; - secfilt[Nsec].dMap = apstats->error; - secfilt[Nsec].sMap = apstats->sigma; - secfilt[Nsec].NusedAp = Nranking; - } - int minRankAper = (Nranking > 0) ? results->aperData[Nsec].ranking[0] : 10; + int NrankingAPER = MAG_STATS_BY_RANKING (&results->aperData[Nsec], apstats); + if (NrankingAPER) { + secfilt[Nsec].MapChp = apstats->mean; + secfilt[Nsec].dMapChp = apstats->error; + secfilt[Nsec].sMapChp = apstats->sigma; + secfilt[Nsec].NusedAp = NrankingAPER; + } + int minRankAper = (NrankingAPER > 0) ? results->aperData[Nsec].ranking[0] : 10; markMeasureByRanking (&results->aperData[Nsec], measure, minRankAper, ID_MEAS_PHOTOM_APER, ID_MEAS_MASKED_APER); - - StatType *kronstats = &results->kronstats; - Nranking = MAG_STATS_BY_RANKING (&results->kronData[Nsec], kronstats); - if (Nranking) { - secfilt[Nsec].Mkron = kronstats->mean; - secfilt[Nsec].dMkron = kronstats->error; - secfilt[Nsec].sMkron = kronstats->sigma; - secfilt[Nsec].NusedKron = Nranking; - } - int minRankKron = (Nranking > 0) ? results->kronData[Nsec].ranking[0] : 10; - markMeasureByRanking (&results->kronData[Nsec], measure, minRankKron, ID_MEAS_PHOTOM_KRON, ID_MEAS_MASKED_KRON); // does this object appear extended in > 50% of measurements? @@ -518,4 +525,9 @@ // 2) select the BEST detections per filter (regardless of PRIMARY) // 3) apply the zero point and AB->Jy transformations + +// I calculate McalAPER and McalPSF independently for stacks. I use McalAPER for Mkron +// and Map, and McalPSF for Mpsf. Note in StarOps.c:setMcalOutput I am setting +// measure->McalAPER to image->McalPSF for chips and warps, but not stacks + int setMrelAverageStack (Catalog *catalog, int cat, off_t ave, int Nsecfilt, FlatCorrectionTable *flatcorr) { @@ -528,7 +540,5 @@ SecFilt *secfilt = &catalog[0].secfilt[ave*Nsecfilt]; - off_t k; - - float Mcal = 0, Mmos = 0, Mgrid = 0, Finst = 0; + float McalPSF = 0, McalAPER = 0, Mmos = 0, Mgrid = 0; // set the primary projection cell and skycell for this coordinate @@ -546,6 +556,5 @@ int haveStackObject = FALSE; - int Ns; - for (Ns = 0; Ns < Nphotcodes; Ns++) { + for (int Ns = 0; Ns < Nphotcodes; Ns++) { int thisCode = photcodes[Ns][0].code; @@ -556,20 +565,37 @@ int haveStack = FALSE; - float psfQFbest = 0.0; - - off_t stackBestMeasure = -1; - off_t stackPrimaryMeasure = -1; - int isBad = FALSE; int isSuspect = FALSE; - int Nstack = 0; // number for this photcode - int NstackDet = 0; // number for this photcode - + int NstackMeas = 0; // number of stack measurements for this photcode + int NstackDet = 0; // number of stack detections for this photcode (not forced) + + int primaryLevelMax = 0; + off_t primaryEntryMax = -1; + float primaryValueMax = 0.0; + + int bestLevelMax = 0; + off_t bestEntryMax = -1; + float bestValueMax = 0.0; + + int Nprimary = 0; + + // reset all stack-related values for this secfilt: secfilt[Nsec].stackBestOff = -1; secfilt[Nsec].stackPrmryOff = -1; - - off_t meas = measureOffset; - for (k = 0; k < Nmeasure; k++, meas++) { + secfilt[Nsec].FpsfStk = NAN; + secfilt[Nsec].dFpsfStk = NAN; + secfilt[Nsec].FkronStk = NAN; + secfilt[Nsec].dFkronStk = NAN; + secfilt[Nsec].FapStk = NAN; + secfilt[Nsec].dFapStk = NAN; + secfilt[Nsec].MpsfStk = NAN; + secfilt[Nsec].MkronStk = NAN; + secfilt[Nsec].MapStk = NAN; + secfilt[Nsec].Nstack = 0; + secfilt[Nsec].NstackDet = 0; + secfilt[Nsec].flags &= ~ID_SECF_STACK_FLAGS; // reset the stack flags + + for (off_t k = 0; k < Nmeasure; k++) { // only examine gpc1 stack data @@ -581,6 +607,6 @@ if (code->equiv != thisCode) { continue; } - Nstack ++; - if ((measure[k].photFlags2 & 0x00000004) == 0) NstackDet ++; + NstackMeas ++; + if ((measure[k].photFlags2 & 0x00000004) == 0) NstackDet ++; // 0x04 = PM_SOURCE_MODE2_MATCHED // clear this bit for all measurements @@ -590,87 +616,102 @@ haveStack = TRUE; haveStackObject = TRUE; + + // if (measure[k].dbFlags & MEAS_BAD) SKIP_THIS_MEAS_STACK(Nbad); + + int isPrimary = FALSE; - // ** find the PRIMARY measurement - - // if we request the primary (USE_TREE_FOR_PRIMARY), this is true if the measurement is from the - // primary skycell for this position - if (MatchImageSkycellID (meas, cat, tessID, projectID, skycellID)) { - stackPrimaryMeasure = k; + // ** is this a PRIMARY measurement? (there may be more than one) + + // if we request the primary (USE_TREE_FOR_PRIMARY), this is true if the measurement + // is from the primary skycell for this position. (note that MatchImageSkycellID + // requires the entry in the full Measure table, not just this object) + if (MatchImageSkycellID (measureOffset + k, cat, tessID, projectID, skycellID)) { measure[k].dbFlags |= ID_MEAS_STACK_PRIMARY; - secfilt[Nsec].stackPrmryOff = meas; - myAssert (secfilt[Nsec].stackPrmryOff <= catalog[0].Nmeasure, "stackPrmryOff out of range"); - } - - // ** now choose the BEST measurements (may also be PRIMARY) - - // ensure that we at least have a single best measure - if (stackBestMeasure == -1) stackBestMeasure = k; - - // choose the best psfQFperf value for the BEST measurement - if (isfinite(measure[k].psfQFperf) && (measure[k].psfQFperf > psfQFbest)) { - psfQFbest = measure[k].psfQFperf; - stackBestMeasure = k; - } - // ... UNLESS psfQFperf > 0.98 for the primary, in which case just use the primary. - if ((measure[k].dbFlags & ID_MEAS_STACK_PRIMARY) && isfinite(measure[k].psfQFperf) && (measure[k].psfQFperf > 0.98)) { - psfQFbest = 1000; // force this to be the best entry - stackBestMeasure = k; - } - - if (measure[k].dbFlags & MEAS_BAD) SKIP_THIS_MEAS_STACK(Nbad); - + isPrimary = TRUE; + Nprimary ++; + } + + // soften the error floor (can dM be 0.0?) + // XXX EAM : this was int SNvalue -- isfinite(SNvalue) would always be finite + float SNvalue = isfinite(measure[k].dM) ? 1.0 / hypot (measure[k].dM, MIN_ERROR) : NAN; + int psfQFperfAboveLimit = isfinite(SNvalue) && isfinite(measure[k].psfQFperf) && (measure[k].psfQFperf > 0.95); + + // ** determine the BEST level + int bestLevel = 0; + if (!isPrimary && !psfQFperfAboveLimit) bestLevel = 1; + if ( isPrimary && !psfQFperfAboveLimit) bestLevel = 2; + if (!isPrimary && psfQFperfAboveLimit) bestLevel = 3; + if ( isPrimary && psfQFperfAboveLimit) bestLevel = 4; + + // here is the rule for choosing the best value: + float bestValue = (bestLevel > 2) ? SNvalue : measure[k].psfQFperf; + + // if we have not reached this level before, set the new level + if (bestLevelMax < bestLevel) { + bestValueMax = bestValue; + bestLevelMax = bestLevel; + bestEntryMax = k; + } + // if we have reached this level before, set the new value if we beat the old one + if ((bestLevelMax == bestLevel) && (bestValueMax < bestValue)) { + bestValueMax = bestValue; + bestEntryMax = k; + } + myAssert (bestEntryMax > -1, "this should not happen"); + + // ** determine the PRIMARY level + int primaryLevel = 0; + if ( isPrimary && !psfQFperfAboveLimit) primaryLevel = 1; + if ( isPrimary && psfQFperfAboveLimit) primaryLevel = 2; + + // here is the rule for choosing the PRIMARY value: + float primaryValue = 0.0; + if (primaryLevel == 1) { primaryValue = measure[k].psfQFperf; } + if (primaryLevel == 2) { primaryValue = SNvalue; } + + // if we have not reached this level before, set the new level + if (isPrimary && (primaryLevelMax < primaryLevel)) { + primaryValueMax = primaryValue; + primaryLevelMax = primaryLevel; + primaryEntryMax = k; + } + // if we have reached this level before, set the new value if we beat the old one + if (isPrimary && (primaryLevelMax == primaryLevel) && (primaryValueMax < primaryValue)) { + primaryValueMax = primaryValue; + primaryEntryMax = k; + } + } + + // if we do not have a stack measurement for this photcode, skip everything below + if (!haveStack) continue; + + // now we have a BEST and a PRIMARY entry (may be the same entry) + + off_t meas = bestEntryMax; + off_t measSeq = meas + measureOffset; + + myAssert (meas >= 0, "this should not happen"); + + // measurements without an image are either external reference photometry or + // data for which the associated image has not been loaded (probably because of + // overlaps). we only want measurements associated with stack images in this loop + + // match measurement to its image (this is just a check, right?) + if (getImageEntry (measSeq, cat) < 0) { // measurements without an image are either external reference photometry or // data for which the associated image has not been loaded (probably because of - // overlaps). we only want measurements associated with stack images in this loop - - // match measurement to its image (this is just a check, right?) - if (getImageEntry (meas, cat) < 0) { - // measurements without an image are either external reference photometry or - // data for which the associated image has not been loaded (probably because of - // overlaps). Msys + measure.Mcal is our best guess of the true magnitude - Mmos = Mgrid = 0; - Mcal = measure[k].Mcal; // check that this is zero for loaded REF value - } else { - Mcal = getMcal_alt (meas, cat, flatcorr, measure[k].Xccd, measure[k].Yccd); - if (isnan(Mcal)) SKIP_THIS_MEAS_STACK(Ncal); - Mmos = getMmos (meas, cat); - if (isnan(Mmos)) SKIP_THIS_MEAS_STACK(Nmos); - Mgrid = getMgrid (meas, cat); - if (isnan(Mgrid)) SKIP_THIS_MEAS_STACK(Ngrid); - } - - // NOTE: negative and insignificant flux values are allowed, but not NAN flux values - Finst = PhotFluxInst (&measure[k], MAG_CLASS_PSF); - if (isnan(Finst)) SKIP_THIS_MEAS_STACK(Ninst); - - // data quality assessment - isBad |= (measure[k].photFlags & code->photomBadMask); - isBad |= (measure[k].psfQF < 0.85); - isBad |= isnan(measure[k].psfQF); - isBad |= measure[k].dM > 0.2; // S/N < 5.0 - - isSuspect |= (measure[k].photFlags & code->photomPoorMask); - isSuspect |= (measure[k].psfQFperf < 0.85); - } - - if (!haveStack) continue; - - if (!isSuspect && !isBad) { - NstackGood ++; - } - if (isSuspect && !isBad) { - NstackSuspect ++; - } - - // measurements which are bad will not have a valid stack entry and are skipped - k = (stackBestMeasure >= 0) ? stackBestMeasure : stackPrimaryMeasure; - if (k < 0) continue; - - // we are now populating stackDetectID not stack Image ID in secfilt - // ID = (stackPrimaryMeasureMin >= 0) ? stackPrimaryIDmin : stackCenterIDmin; - // ID = measure[k].extID; // for the stack, this is the stackDetectID - - // get the zero point for the selected image - float zp = PhotZeroPoint (&measure[k], &average[0], &secfilt[0]) - (Mcal + Mmos + Mgrid); + // overlaps). Msys + measure.Mcal is our best guess of the true magnitude + McalPSF = measure[meas].McalPSF; // check that this is zero for loaded REF value + McalAPER = McalPSF; // check that this is zero for loaded REF value + Mmos = 0.0; + Mgrid = 0.0; + } else { + McalPSF = getMcal (measSeq, cat, MAG_CLASS_PSF); + McalAPER = USE_MCAL_PSF_FOR_STACK_APER ? getMcal (measSeq, cat, MAG_CLASS_PSF) : getMcal (measSeq, cat, MAG_CLASS_KRON); + Mmos = getMmos (measSeq, cat); + Mgrid = getMgrid (measSeq, cat); + } + + // ** Here is the math to relate mag,zp to flux // flux_cgs : erg sec^1 cm^-2 Hz^-1 @@ -703,46 +744,81 @@ // flux_Jy = flux_inst * ten(-0.4*ZP + 3.56) + // get the zero point for the selected image + // Use a different zero point for the PSF-like and APERTURE-like magnitudes + float zpPSF = PhotZeroPoint (&measure[meas], &average[0], &secfilt[0]) - (McalPSF + Mmos + Mgrid); + float zpAPER = PhotZeroPoint (&measure[meas], &average[0], &secfilt[0]) - (McalAPER + Mmos + Mgrid); + // zpFactor to go from instrumental flux to Janskies - float zpFactor = pow(10.0, -0.4*zp + 3.56); + float zpFactorPSF = pow(10.0, -0.4*zpPSF + 3.56); + float zpFactorAPER = pow(10.0, -0.4*zpAPER + 3.56); // need to put in AB mag factor to get to Janskies (or uJy?) - secfilt[Nsec].FpsfStk = zpFactor * measure[k].FluxPSF; - secfilt[Nsec].dFpsfStk = zpFactor * measure[k].dFluxPSF; - secfilt[Nsec].FkronStk = zpFactor * measure[k].FluxKron; - secfilt[Nsec].dFkronStk = zpFactor * measure[k].dFluxKron; - secfilt[Nsec].FapStk = zpFactor * measure[k].FluxAp; + secfilt[Nsec].FpsfStk = zpFactorPSF * measure[meas].FluxPSF; + secfilt[Nsec].dFpsfStk = zpFactorPSF * measure[meas].dFluxPSF; + secfilt[Nsec].FkronStk = zpFactorAPER * measure[meas].FluxKron; + secfilt[Nsec].dFkronStk = zpFactorAPER * measure[meas].dFluxKron; + secfilt[Nsec].FapStk = zpFactorAPER * measure[meas].FluxAp; // NOTE: for PV3, apFluxErr is broken (see pmSourcePhotometry.c:245). we are going to // use PSF flux error instead here: - // secfilt[Nsec].dFapStk = zpFactor * measure[k].dFluxAp; - secfilt[Nsec].dFapStk = zpFactor * measure[k].dFluxPSF; + // secfilt[Nsec].dFapStk = zpFactorAPER * measure[meas].dFluxAp; + secfilt[Nsec].dFapStk = zpFactorAPER * measure[meas].dFluxPSF; // Jy to AB mags - secfilt[Nsec].MpsfStk = (measure[k].FluxPSF > 0.0) ? 8.9 - 2.5*log10(secfilt[Nsec].FpsfStk) : NAN; - secfilt[Nsec].MkronStk = (measure[k].FluxKron > 0.0) ? 8.9 - 2.5*log10(secfilt[Nsec].FkronStk) : NAN; - secfilt[Nsec].MapStk = (measure[k].FluxAp > 0.0) ? 8.9 - 2.5*log10(secfilt[Nsec].FapStk) : NAN; - - secfilt[Nsec].stackBestOff = k + measureOffset; + secfilt[Nsec].MpsfStk = (measure[meas].FluxPSF > 0.0) ? 8.9 - 2.5*log10(secfilt[Nsec].FpsfStk) : NAN; + secfilt[Nsec].MkronStk = (measure[meas].FluxKron > 0.0) ? 8.9 - 2.5*log10(secfilt[Nsec].FkronStk) : NAN; + secfilt[Nsec].MapStk = (measure[meas].FluxAp > 0.0) ? 8.9 - 2.5*log10(secfilt[Nsec].FapStk) : NAN; + + // record the measurement which gave the best value + secfilt[Nsec].stackBestOff = (bestEntryMax == -1) ? -1 : bestEntryMax + measureOffset; myAssert (secfilt[Nsec].stackBestOff <= catalog[0].Nmeasure, "stackBestOff out of range"); - secfilt[Nsec].Nstack = Nstack; + // record the selected primary measurement + secfilt[Nsec].stackPrmryOff = (primaryEntryMax == -1) ? -1 : primaryEntryMax + measureOffset; + myAssert (secfilt[Nsec].stackPrmryOff <= catalog[0].Nmeasure, "stackPrmryOff out of range"); + + // this is the selected measurement used by secfilt[] + measure[meas].dbFlags |= ID_MEAS_STACK_PHOT_SRC; + if (Nprimary) { + secfilt[Nsec].flags |= ID_SECF_STACK_PRIMARY; + if (Nprimary > 1) { + secfilt[Nsec].flags |= ID_SECF_STACK_PRIMARY_MULTIPLE; + } + } + + // stack measurement used for 'best' was a detection (not forced from the other bands) + if ((measure[bestEntryMax].photFlags2 & 0x00000004) == 0) { + secfilt[Nsec].flags |= ID_SECF_STACK_BESTDET; + } + + // stack measurement used for 'primary' was a detection (not forced from the other bands) + if ((primaryEntryMax >= 0) && ((measure[primaryEntryMax].photFlags2 & 0x00000004) == 0)) { + secfilt[Nsec].flags |= ID_SECF_STACK_PRIMDET; + } + + secfilt[Nsec].flags |= ID_SECF_HAS_PS1_STACK; + + // NOTE: negative and insignificant flux values are allowed, but not NAN flux values + // float Finst = PhotFluxInst (&measure[meas], MAG_CLASS_PSF); + + // data quality assessment + isBad |= (measure[meas].photFlags & photcodes[Ns][0].photomBadMask); + isBad |= (measure[meas].psfQF < 0.85); + isBad |= isnan(measure[meas].psfQF); + isBad |= measure[meas].dM > 0.2; // S/N < 5.0 + + isSuspect |= (measure[meas].photFlags & photcodes[Ns][0].photomPoorMask); + isSuspect |= (measure[meas].psfQFperf < 0.85); + + if (!isSuspect && !isBad) { + NstackGood ++; + } + if (isSuspect && !isBad) { + NstackSuspect ++; + } + + secfilt[Nsec].Nstack = NstackMeas; secfilt[Nsec].NstackDet = NstackDet; - // this is the measurement used by secfilt[] - measure[k].dbFlags |= ID_MEAS_STACK_PHOT_SRC; - if (k == stackPrimaryMeasure) { - secfilt[Nsec].flags |= ID_SECF_STACK_PRIMARY; - } - - // stack measurement used for 'best' was a detection (not forced from the other bands) - if ((measure[k].photFlags2 & 0x00000004) == 0) { - secfilt[Nsec].flags |= ID_SECF_STACK_BESTDET; - } - // stack measurement used for 'primary' was a detection (not forced from the other bands) - if ((stackPrimaryMeasure >= 0) && ((measure[stackPrimaryMeasure].photFlags2 & 0x00000004) == 0)) { - secfilt[Nsec].flags |= ID_SECF_STACK_PRIMDET; - } - - secfilt[Nsec].flags |= ID_SECF_HAS_PS1_STACK; } // Nsecfilt loop @@ -755,5 +831,5 @@ int PrimaryIsBest = TRUE; - for (Ns = 0; Ns < Nphotcodes; Ns++) { + for (int Ns = 0; Ns < Nphotcodes; Ns++) { if (!(secfilt[Ns].flags & ID_SECF_HAS_PS1_STACK)) continue; // no stack detection in PS1, nothing is best if (secfilt[Ns].flags & ID_SECF_STACK_PRIMARY) continue; // primary stack detection is best @@ -780,4 +856,9 @@ // * no grid, no mosaic, no 2MASS, no SYNTH, no Ubercal, no flatcorr // analysis is done on flux, not mags (as the faintest objects will be nearly insignificant) + +// Although I calculate McalAPER for exposures, I am only using McalPSF for warps.. +// Note in StarOps.c:setMcalOutput I am setting measure->McalAPER to image->McalPSF for +// chips and warps, but not stacks + int setMrelAverageForcedWarp (Catalog *catalog, int cat, off_t ave, int Nsecfilt, FlatCorrectionTable *flatcorr, SetMrelInfo *results) { OHANA_UNUSED_PARAM(flatcorr); @@ -857,9 +938,7 @@ // data for which the associated image has not been loaded (probably because of // overlaps). Msys + measure.Mcal is our best guess of the true magnitude - Mcal = measure[k].Mcal; // check that this is zero for loaded REF value + Mcal = measure[k].McalPSF; // check that this is zero for loaded REF value } else { - // use getMcal not getMcal_alt? - Mcal = getMcal_alt (meas, cat, NULL, measure[k].Xccd, measure[k].Yccd); - // Mcal = getMcal (meas, cat); + Mcal = getMcal (meas, cat, MAG_CLASS_PSF); if (isnan(Mcal)) SKIP_THIS_MEAS(Ncal); } Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/setMrelFinal.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/setMrelFinal.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/setMrelFinal.c (revision 40477) @@ -65,10 +65,12 @@ if (catalog[0].measure[m].dbFlags & ID_MEAS_PHOTOM_UBERCAL) { if (!KEEP_UBERCAL) { - catalog[0].measure[m].Mcal = 0.0; + catalog[0].measure[m].McalPSF = 0.0; + catalog[0].measure[m].McalAPER = 0.0; catalog[0].measure[m].dbFlags &= ~ID_MEAS_PHOTOM_UBERCAL; } } else { if (RESET_ZEROPTS && (getImageEntry (m, 0) >= 0)) { - catalog[0].measure[m].Mcal = 0.0; + catalog[0].measure[m].McalPSF = 0.0; + catalog[0].measure[m].McalAPER = 0.0; } } @@ -105,5 +107,5 @@ ALLOCATE (catalog[0].measureRank, char, catalog[0].Nmeasure); setMeasureRank (catalog); - setMrelOutput (catalog, 1, flatcorr); // sets the values secfilt.M = + setMrelOutput (catalog, 1, flatcorr); // sets the values secfilt.MpsfChp = setMcalOutput (catalog, 1, flatcorr); // sets measure.Mcal = image.Mcal @@ -226,5 +228,5 @@ off_t Nim = getImageEntry (m, 0); if (Nim > -1) { - if (isnan(getMcal (m, 0, flatcorr, catalog))) goto skip; + if (isnan(getMcal (m, 0, MAG_CLASS_PSF))) goto skip; if (isnan(getMmos (m, 0))) goto skip; } Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/share_image_mags.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/share_image_mags.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/share_image_mags.c (revision 40477) @@ -74,8 +74,9 @@ continue; } - images[seq].Mcal = image_mags[i].Mcal; + images[seq].McalPSF = image_mags[i].McalPSF; + images[seq].McalAPER = image_mags[i].McalAPER; images[seq].dMcal = image_mags[i].dMcal; images[seq].dMagSys = image_mags[i].dMagSys; - images[seq].Xm = image_mags[i].Xm; + images[seq].McalChiSq = image_mags[i].McalChiSq; images[seq].nFitPhotom = image_mags[i].nFitPhotom; images[seq].flags = image_mags[i].flags; @@ -91,8 +92,9 @@ int set_image_mags (ImageMag *image_mags, Image *image) { - image_mags->Mcal = image->Mcal; + image_mags->McalPSF = image->McalPSF; + image_mags->McalAPER = image->McalAPER; image_mags->dMcal = image->dMcal; image_mags->dMagSys = image->dMagSys; - image_mags->Xm = image->Xm; + image_mags->McalChiSq = image->McalChiSq; image_mags->nFitPhotom = image->nFitPhotom; image_mags->flags = image->flags; Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/share_mean_mags.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/share_mean_mags.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/share_mean_mags.c (revision 40477) @@ -111,5 +111,5 @@ if (Nsec < 0) continue; - catalog[catSeq].secfilt[objSeq*Nsecfilt + Nsec].M = meanmags[i].M; + catalog[catSeq].secfilt[objSeq*Nsecfilt + Nsec].MpsfChp = meanmags[i].M; } free (meanmags); @@ -122,6 +122,6 @@ int set_mean_mags (MeanMag *meanmags, AverageTiny *average, SecFilt *secfilt, int Nsec) { - meanmags->M = secfilt->M; - meanmags->dM = secfilt->dM; + meanmags->M = secfilt->MpsfChp; + meanmags->dM = secfilt->dMpsfChp; meanmags->Mchisq = secfilt->Mchisq; meanmags->Nsec = Nsec; // key to secfilt entry Index: /branches/czw_branch/20170908/Ohana/src/relphot/src/synthetic_mags.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/relphot/src/synthetic_mags.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/relphot/src/synthetic_mags.c (revision 40477) @@ -37,6 +37,6 @@ // (r - i > 0.5) : w = r + 0.268 - 0.435 (r-i) - 0.078(r-i)^2 - float Mr = secfilt[NSr].M; - float Mi = secfilt[NSi].M; + float Mr = secfilt[NSr].MpsfChp; + float Mi = secfilt[NSi].MpsfChp; if (!isfinite(Mr)) return FALSE; Index: /branches/czw_branch/20170908/Ohana/src/tools/src/fields.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/tools/src/fields.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/tools/src/fields.c (revision 40477) @@ -96,4 +96,5 @@ } } + // we are checking each extension to see if it matches the regex. if (!regexec (&preg, extname, 0, NULL, 0)) { GotField &= print_fields (filename, extname, &header, argc, argv); @@ -105,4 +106,8 @@ Nextend ++; + // this implementation is very inefficient! + // since we are checking each header for a regex, we should + // loop over all extensions once. the function below reads the + // entire file a second time GotFile &= gfits_read_Xheader (filename, &header, Nextend); continue; Index: /branches/czw_branch/20170908/Ohana/src/uniphot/include/setphot.h =================================================================== --- /branches/czw_branch/20170908/Ohana/src/uniphot/include/setphot.h (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/uniphot/include/setphot.h (revision 40477) @@ -11,5 +11,6 @@ typedef struct { - float Mcal; + float McalPSF; + float McalAPER; float dMcal; unsigned int imageID; Index: /branches/czw_branch/20170908/Ohana/src/uniphot/src/ImageSubset.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/uniphot/src/ImageSubset.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/uniphot/src/ImageSubset.c (revision 40477) @@ -56,9 +56,10 @@ char type[16]; - GET_COLUMN (Mcal, "MCAL", float); - GET_COLUMN (dMcal, "MCAL_ERR", float); - GET_COLUMN (imageID, "IMAGE_ID", int); - GET_COLUMN (map, "PHOTOM_MAP", int); - GET_COLUMN (flags, "FLAGS", int); + GET_COLUMN (McalPSF, "MCAL_PSF", float); + GET_COLUMN (McalAPER, "MCAL_APER", float); + GET_COLUMN (dMcal, "MCAL_ERR", float); + GET_COLUMN (imageID, "IMAGE_ID", int); + GET_COLUMN (map, "PHOTOM_MAP", int); + GET_COLUMN (flags, "FLAGS", int); // XXX free the fits table data here @@ -69,10 +70,12 @@ image[i].photom_map_id = map[i]; image[i].flags = flags[i]; - image[i].Mcal = Mcal[i]; + image[i].McalPSF = McalPSF[i]; + image[i].McalAPER = McalAPER[i]; image[i].dMcal = dMcal[i]; } fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow); - free (Mcal); + free (McalPSF); + free (McalAPER); free (dMcal); free (imageID); @@ -108,31 +111,32 @@ gfits_create_table (&theader, &ftable); - float *Mcal, *dMcal; - unsigned int *imageID, *map, *flags; - // create intermediate storage arrays - ALLOCATE (Mcal, float, Nimage); - ALLOCATE (dMcal, float, Nimage); - ALLOCATE (imageID, unsigned int, Nimage); - ALLOCATE (map, unsigned int, Nimage); - ALLOCATE (flags, unsigned int, Nimage); + ALLOCATE_PTR (McalPSF, float, Nimage); + ALLOCATE_PTR (McalAPER, float, Nimage); + ALLOCATE_PTR (dMcal, float, Nimage); + ALLOCATE_PTR (imageID, unsigned int, Nimage); + ALLOCATE_PTR (map, unsigned int, Nimage); + ALLOCATE_PTR (flags, unsigned int, Nimage); // assign the storage arrays for (i = 0; i < Nimage; i++) { - imageID[i] = image[i].imageID; - map[i] = image[i].photom_map_id; - flags[i] = image[i].flags; - Mcal[i] = image[i].Mcal; - dMcal[i] = image[i].dMcal; + imageID[i] = image[i].imageID; + map[i] = image[i].photom_map_id; + flags[i] = image[i].flags; + McalPSF[i] = image[i].McalPSF; + McalAPER[i] = image[i].McalAPER; + dMcal[i] = image[i].dMcal; } // add the columns to the output array - gfits_set_bintable_column (&theader, &ftable, "MCAL", Mcal, Nimage); - gfits_set_bintable_column (&theader, &ftable, "MCAL_ERR", dMcal, Nimage); - gfits_set_bintable_column (&theader, &ftable, "IMAGE_ID", imageID, Nimage); - gfits_set_bintable_column (&theader, &ftable, "PHOTOM_MAP", map, Nimage); - gfits_set_bintable_column (&theader, &ftable, "FLAGS", flags, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MCAL_PSF", McalPSF, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MCAL_APER", McalAPER, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MCAL_ERR", dMcal, Nimage); + gfits_set_bintable_column (&theader, &ftable, "IMAGE_ID", imageID, Nimage); + gfits_set_bintable_column (&theader, &ftable, "PHOTOM_MAP", map, Nimage); + gfits_set_bintable_column (&theader, &ftable, "FLAGS", flags, Nimage); - free (Mcal); + free (McalPSF); + free (McalAPER); free (dMcal); free (imageID); @@ -174,5 +178,6 @@ image[i].photom_map_id = subset[i].photom_map_id; image[i].flags = subset[i].flags ; - image[i].Mcal = subset[i].Mcal ; + image[i].McalPSF = subset[i].McalPSF ; + image[i].McalAPER = subset[i].McalAPER ; image[i].dMcal = subset[i].dMcal ; } Index: /branches/czw_branch/20170908/Ohana/src/uniphot/src/dumpresult.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/uniphot/src/dumpresult.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/uniphot/src/dumpresult.c (revision 40477) @@ -2,5 +2,5 @@ void dumpresult () { - short Mcal, Mgrp, Mset; + float Mcal, Mgrp, Mset; FILE *f; char outfile[64]; @@ -13,5 +13,5 @@ if (sgroup[i].image[j][0].code & IMAGE_BAD) continue; tgrp = (Group *) sgroup[i].imlink[j][0].tgroup; - Mcal = sgroup[i].image[j][0].Mcal; + Mcal = sgroup[i].image[j][0].McalPSF; Mset = sgroup[i].M; Mgrp = tgrp[0].M; Index: /branches/czw_branch/20170908/Ohana/src/uniphot/src/fit_groups.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/uniphot/src/fit_groups.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/uniphot/src/fit_groups.c (revision 40477) @@ -18,5 +18,5 @@ if (tgroup[i].image[j][0].flags & IMAGE_BAD) continue; sgroup = (Group *) tgroup[i].imlink[j][0].sgroup; - Mcal = tgroup[i].image[j][0].Mcal; + Mcal = tgroup[i].image[j][0].McalPSF; Mgrp = sgroup[0].M; mlist[Nlist] = (Mcal - Mgrp); @@ -58,5 +58,5 @@ if (sgroup[i].image[j][0].flags & IMAGE_BAD) continue; tgroup = (Group *) sgroup[i].imlink[j][0].tgroup; - Mcal = sgroup[i].image[j][0].Mcal; + Mcal = sgroup[i].image[j][0].McalPSF; Mgrp = tgroup[0].M; mlist[Nlist] = (Mcal - Mgrp); Index: /branches/czw_branch/20170908/Ohana/src/uniphot/src/match_zpts_to_images.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/uniphot/src/match_zpts_to_images.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/uniphot/src/match_zpts_to_images.c (revision 40477) @@ -41,7 +41,8 @@ if (RESET) { for (i = 0; i < Nimage; i++) { - image[i].Mcal = 0.0; - image[i].dMcal = NAN; - image[i].flags &= ~ID_IMAGE_PHOTOM_NOCAL; // clear the NOCAL flag + image[i].McalPSF = 0.0; + image[i].McalAPER = 0.0; + image[i].dMcal = NAN; + image[i].flags &= ~ID_IMAGE_PHOTOM_NOCAL; // clear the NOCAL flag if (UBERCAL) { image[i].flags &= ~ID_IMAGE_PHOTOM_UBERCAL; // clear the UBERCAL flag @@ -107,8 +108,10 @@ // UBERCAL includes 2.5log(exptime) + K*airmass in the zero point if (UBERCAL) { - image[Ni].Mcal = SCALE*code[0].C - zpts[Nz].zpt + 2.5*log10(image[Ni].exptime) + code[0].K*(image[Ni].secz - 1.000); - myAssert (isfinite(image[Ni].Mcal), "oops, ubercal made a nan image"); + image[Ni].McalPSF = SCALE*code[0].C - zpts[Nz].zpt + 2.5*log10(image[Ni].exptime) + code[0].K*(image[Ni].secz - 1.000); + image[Ni].McalAPER = SCALE*code[0].C - zpts[Nz].zpt + 2.5*log10(image[Ni].exptime) + code[0].K*(image[Ni].secz - 1.000); + myAssert (isfinite(image[Ni].McalPSF), "oops, ubercal made a nan image"); } else { - image[Ni].Mcal = SCALE*code[0].C - zpts[Nz].zpt; + image[Ni].McalPSF = SCALE*code[0].C - zpts[Nz].zpt; + image[Ni].McalAPER = SCALE*code[0].C - zpts[Nz].zpt; } @@ -116,5 +119,6 @@ float offset = apply_zpt_offset (code[0].equiv); assert (isfinite(offset)); - image[Ni].Mcal += offset; + image[Ni].McalPSF += offset; + image[Ni].McalAPER += offset; image[Ni].dMcal = zpts[Nz].zpt_err; Index: /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_catalog_setastrom.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_catalog_setastrom.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_catalog_setastrom.c (revision 40477) @@ -153,6 +153,6 @@ case 101: case 102: { - float gmag = secfilt[Nsec_g].M; - float imag = secfilt[Nsec_i].M; + float gmag = secfilt[Nsec_g].MpsfChp; + float imag = secfilt[Nsec_i].MpsfChp; dColor = average->refColorBlue - (gmag - imag); break; @@ -160,6 +160,6 @@ case 103: case 104: { - float zmag = secfilt[Nsec_z].M; - float ymag = secfilt[Nsec_y].M; + float zmag = secfilt[Nsec_z].MpsfChp; + float ymag = secfilt[Nsec_y].MpsfChp; dColor = average->refColorRed - (zmag - ymag); break; Index: /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_catalog_setphot.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_catalog_setphot.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_catalog_setphot.c (revision 40477) @@ -39,5 +39,5 @@ if (id < 0) continue; - float Mcal = image[id].Mcal; + float Mcal = image[id].McalPSF; float dMcal = image[id].dMcal; float Mflat = 0.0; @@ -61,8 +61,10 @@ # endif - measure[0].Mcal = Mcal; + myAssert(isfinite(Mcal), "oops: ubercal made a nan"); + + measure[0].McalPSF = Mcal; + measure[0].McalAPER = Mcal; measure[0].Mflat = Mflat; // in the previous version, Mcal_offset (which is added to Mflat) had a negative sign here measure[0].dMcal = dMcal; - myAssert(isfinite(measure[0].Mcal), "oops: ubercal made a nan"); if (RESET) { Index: /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_catalog_uniphot.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_catalog_uniphot.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_catalog_uniphot.c (revision 40477) @@ -14,6 +14,6 @@ for (i = 0; i < catalog[0].Naverage; i++) { - if (!isnan(catalog[0].secfilt[i*Nsecfilt+Nsec].M)) { - catalog[0].secfilt[i*Nsecfilt+Nsec].M += sgroup[0].M; + if (!isnan(catalog[0].secfilt[i*Nsecfilt+Nsec].MpsfChp)) { + catalog[0].secfilt[i*Nsecfilt+Nsec].MpsfChp += sgroup[0].M; } @@ -24,5 +24,6 @@ if (code[0].type != PHOT_DEP) continue; if (code[0].equiv != photcode[0].code) continue; - catalog[0].measure[m].Mcal -= sgroup[0].M; + catalog[0].measure[m].McalPSF -= sgroup[0].M; + catalog[0].measure[m].McalAPER -= sgroup[0].M; found ++; } Index: /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_dvo_uniphot.c =================================================================== --- /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_dvo_uniphot.c (revision 40476) +++ /branches/czw_branch/20170908/Ohana/src/uniphot/src/update_dvo_uniphot.c (revision 40477) @@ -37,5 +37,6 @@ for (i = 0; i < Nsgroup; i++) { for (j = 0; j < sgroup[i].Nimage; j++) { - sgroup[i].image[j][0].Mcal -= sgroup[i].M; + sgroup[i].image[j][0].McalPSF -= sgroup[i].M; + sgroup[i].image[j][0].McalAPER -= sgroup[i].M; } }