### Put Perl modules in ~/perl/
### Statistics::Descriptive (CPAN)
### PS-IPP-Metadata (CVS)
### PS-IPP-Metadata-Config (CVS)

setenv PERL5LIB $HOME/perl/


### Set up the tables

price@alala:/home/panstarrs/price/ipp/ippTools/pXtools>pxadmin -create
price@alala:/home/panstarrs/price/ipp/ippTools/pXtools>pxadmin -recreate
*** delete the P2 tables? ***
*** to delete the tables, answer YES, and give password ***
*** WARNING: this action is permanent ***
 
delete the P2 tables (YES/[n]): YES
enter dbh connection password: ipp

### Inject everything into the DB

pxinject -newExp -exp_id 850131b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850132b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850133b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850403b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850539b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850540b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850541b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850647b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850648b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850649b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850650b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850651b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850652b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850653b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850654b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850655b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850723b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850822b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850935b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 850936b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1

pxinject -newImfile -exp_id 850131b -class fpa -class_id mc -uri 850131b.fits
pxinject -newImfile -exp_id 850132b -class fpa -class_id mc -uri 850132b.fits
pxinject -newImfile -exp_id 850133b -class fpa -class_id mc -uri 850133b.fits
pxinject -newImfile -exp_id 850403b -class fpa -class_id mc -uri 850403b.fits
pxinject -newImfile -exp_id 850539b -class fpa -class_id mc -uri 850539b.fits
pxinject -newImfile -exp_id 850540b -class fpa -class_id mc -uri 850540b.fits
pxinject -newImfile -exp_id 850541b -class fpa -class_id mc -uri 850541b.fits
pxinject -newImfile -exp_id 850647b -class fpa -class_id mc -uri 850647b.fits
pxinject -newImfile -exp_id 850648b -class fpa -class_id mc -uri 850648b.fits
pxinject -newImfile -exp_id 850649b -class fpa -class_id mc -uri 850649b.fits
pxinject -newImfile -exp_id 850650b -class fpa -class_id mc -uri 850650b.fits
pxinject -newImfile -exp_id 850651b -class fpa -class_id mc -uri 850651b.fits
pxinject -newImfile -exp_id 850652b -class fpa -class_id mc -uri 850652b.fits
pxinject -newImfile -exp_id 850653b -class fpa -class_id mc -uri 850653b.fits
pxinject -newImfile -exp_id 850654b -class fpa -class_id mc -uri 850654b.fits
pxinject -newImfile -exp_id 850655b -class fpa -class_id mc -uri 850655b.fits
pxinject -newImfile -exp_id 850723b -class fpa -class_id mc -uri 850723b.fits
pxinject -newImfile -exp_id 850822b -class fpa -class_id mc -uri 850822b.fits
pxinject -newImfile -exp_id 850935b -class fpa -class_id mc -uri 850935b.fits
pxinject -newImfile -exp_id 850936b -class fpa -class_id mc -uri 850936b.fits

### Look for phase 0 imfiles:

price@alala:/data/alala/price/megacam>p0tool -pendingimfile -simple
850131b fpa mc 850131b.fits
850132b fpa mc 850132b.fits
850133b fpa mc 850133b.fits
850403b fpa mc 850403b.fits
850539b fpa mc 850539b.fits
850540b fpa mc 850540b.fits
850541b fpa mc 850541b.fits
850647b fpa mc 850647b.fits
850648b fpa mc 850648b.fits
850649b fpa mc 850649b.fits
850650b fpa mc 850650b.fits
850651b fpa mc 850651b.fits
850652b fpa mc 850652b.fits
850653b fpa mc 850653b.fits
850654b fpa mc 850654b.fits
850655b fpa mc 850655b.fits
850723b fpa mc 850723b.fits
850822b fpa mc 850822b.fits
850935b fpa mc 850935b.fits
850936b fpa mc 850936b.fits

### Run the phase 0 imfile script:

~/ipp/helpers/phase0imfile.pl 850131b mc 850131b.fits
~/ipp/helpers/phase0imfile.pl 850132b mc 850132b.fits
~/ipp/helpers/phase0imfile.pl 850133b mc 850133b.fits
~/ipp/helpers/phase0imfile.pl 850403b mc 850403b.fits
~/ipp/helpers/phase0imfile.pl 850539b mc 850539b.fits
~/ipp/helpers/phase0imfile.pl 850540b mc 850540b.fits
~/ipp/helpers/phase0imfile.pl 850541b mc 850541b.fits
~/ipp/helpers/phase0imfile.pl 850647b mc 850647b.fits
~/ipp/helpers/phase0imfile.pl 850648b mc 850648b.fits
~/ipp/helpers/phase0imfile.pl 850649b mc 850649b.fits
~/ipp/helpers/phase0imfile.pl 850650b mc 850650b.fits
~/ipp/helpers/phase0imfile.pl 850651b mc 850651b.fits
~/ipp/helpers/phase0imfile.pl 850652b mc 850652b.fits
~/ipp/helpers/phase0imfile.pl 850653b mc 850653b.fits
~/ipp/helpers/phase0imfile.pl 850654b mc 850654b.fits
~/ipp/helpers/phase0imfile.pl 850655b mc 850655b.fits
~/ipp/helpers/phase0imfile.pl 850723b mc 850723b.fits
~/ipp/helpers/phase0imfile.pl 850822b mc 850822b.fits
~/ipp/helpers/phase0imfile.pl 850935b mc 850935b.fits
~/ipp/helpers/phase0imfile.pl 850936b mc 850936b.fits

### Look for phase 0 exposures:

price@alala:/data/alala/price/megacam>p0tool -pendingexp -simple
850131b MegaCam CFHT bias 1
850132b MegaCam CFHT bias 1
850133b MegaCam CFHT bias 1
850403b MegaCam CFHT bias 1
850539b MegaCam CFHT bias 1
850540b MegaCam CFHT bias 1
850541b MegaCam CFHT bias 1
850647b MegaCam CFHT bias 1
850648b MegaCam CFHT bias 1
850649b MegaCam CFHT bias 1
850650b MegaCam CFHT bias 1
850651b MegaCam CFHT bias 1
850652b MegaCam CFHT bias 1
850653b MegaCam CFHT bias 1
850654b MegaCam CFHT bias 1
850655b MegaCam CFHT bias 1
850723b MegaCam CFHT bias 1
850822b MegaCam CFHT bias 1
850935b MegaCam CFHT bias 1
850936b MegaCam CFHT bias 1


### Run the phase 0 exposure script:

~/ipp/helpers/phase0exp.pl 850131b
~/ipp/helpers/phase0exp.pl 850132b
~/ipp/helpers/phase0exp.pl 850133b
~/ipp/helpers/phase0exp.pl 850403b
~/ipp/helpers/phase0exp.pl 850539b
~/ipp/helpers/phase0exp.pl 850540b
~/ipp/helpers/phase0exp.pl 850541b
~/ipp/helpers/phase0exp.pl 850647b
~/ipp/helpers/phase0exp.pl 850648b
~/ipp/helpers/phase0exp.pl 850649b
~/ipp/helpers/phase0exp.pl 850650b
~/ipp/helpers/phase0exp.pl 850651b
~/ipp/helpers/phase0exp.pl 850652b
~/ipp/helpers/phase0exp.pl 850653b
~/ipp/helpers/phase0exp.pl 850654b
~/ipp/helpers/phase0exp.pl 850655b
~/ipp/helpers/phase0exp.pl 850723b
~/ipp/helpers/phase0exp.pl 850822b
~/ipp/helpers/phase0exp.pl 850935b
~/ipp/helpers/phase0exp.pl 850936b

### Start a bias detrend run

price@alala:/data/alala/price/megacam>dettool -definebyquery -det_type bias -camera mc

detRun MULTI #
 
detRun  METADATA
   det_id           S32       1
   iteration        S32       0
   det_type         STR       bias
END
 


### Get list of pending raw images:


price@alala:/data/alala/price/megacam>dettool -raw -simple
1 bias 850131b fpa mc 850131b.fits BIAS r.MP9601 1.656000 5.496118 0.349071 0.000000 1155.934692 107.037283 46.214947 37.160000 105.480000 -133.067001 0.000000
1 bias 850132b fpa mc 850132b.fits BIAS r.MP9601 1.656000 5.499327 0.349071 0.000000 1155.431957 107.464479 46.207451 37.160000 105.480000 -133.067001 0.000000
1 bias 850133b fpa mc 850133b.fits BIAS r.MP9601 1.656000 5.502535 0.349071 0.000000 1154.871040 107.485792 46.227965 37.160000 105.480000 -133.067001 0.000000
1 bias 850403b fpa mc 850403b.fits BIAS r.MP9601 1.000000 2.184885 0.349065 0.000000 1156.454996 105.813700 46.699662 89.590000 272.850000 -133.100006 0.000000
1 bias 850539b fpa mc 850539b.fits BIAS HaOFF.MP7604 1.000000 2.928339 0.349071 0.000000 1155.991272 105.710902 46.830444 -9999.900000 -9999.900000 -133.100006 0.000000
1 bias 850540b fpa mc 850540b.fits BIAS HaOFF.MP7604 1.000000 2.934318 0.349071 0.000000 1156.668537 106.228856 46.638641 -9999.900000 -9999.900000 -133.132996 0.000000
1 bias 850541b fpa mc 850541b.fits BIAS NONE 1.035000 1.539753 0.349058 0.000000 1157.512387 105.972075 46.692062 75.000000 272.590000 -133.233002 0.000000
1 bias 850647b fpa mc 850647b.fits BIAS i.MP9701 1.000000 5.661125 0.349065 0.000000 1159.236140 105.088919 46.581451 89.640000 273.780000 -133.132996 0.000000
1 bias 850648b fpa mc 850648b.fits BIAS i.MP9701 1.000000 5.664260 0.349065 0.000000 1159.642160 105.172955 46.462718 89.640000 273.780000 -133.132996 0.000000
1 bias 850649b fpa mc 850649b.fits BIAS i.MP9701 1.000000 5.667395 0.349065 0.000000 1159.877803 105.578676 46.323573 89.640000 273.780000 -133.132996 0.000000
1 bias 850650b fpa mc 850650b.fits BIAS i.MP9701 1.000000 5.670604 0.349065 0.000000 1159.296260 105.885271 46.306679 89.640000 273.780000 -133.132996 0.000000
1 bias 850651b fpa mc 850651b.fits BIAS i.MP9701 1.000000 5.673812 0.349065 0.000000 1159.130751 106.263144 46.198493 89.640000 273.780000 -133.132996 0.000000
1 bias 850652b fpa mc 850652b.fits BIAS i.MP9701 1.000000 5.677020 0.349065 0.000000 1158.711761 106.354676 46.160953 89.640000 273.780000 -133.100006 0.000000
1 bias 850653b fpa mc 850653b.fits BIAS i.MP9701 1.000000 5.680301 0.349065 0.000000 1157.834599 106.646493 46.197773 89.640000 273.780000 -133.100006 0.000000
1 bias 850654b fpa mc 850654b.fits BIAS i.MP9701 1.000000 5.683510 0.349065 0.000000 1157.518896 106.736919 46.124295 89.640000 273.780000 -133.067001 0.000000
1 bias 850655b fpa mc 850655b.fits BIAS i.MP9701 1.000000 5.686791 0.349065 0.000000 1156.908897 107.101147 46.126100 89.640000 273.780000 -133.167007 0.000000
1 bias 850723b fpa mc 850723b.fits BIAS i.MP9701 1.000000 1.681629 0.349065 0.000000 1156.671385 105.943337 46.696208 89.640000 273.780000 -133.167007 0.000000
1 bias 850822b fpa mc 850822b.fits BIAS r.MP9601 1.318000 1.317600 0.349039 0.000000 1156.904900 105.906221 46.700064 49.330000 156.180000 -133.167007 0.000000
1 bias 850935b fpa mc 850935b.fits BIAS i.MP9701 1.222000 0.670460 0.349065 0.000000 1156.704276 105.995095 46.740883 54.930000 102.740000 -133.167007 0.000000
1 bias 850936b fpa mc 850936b.fits BIAS r.MP9601 1.222000 2.741378 0.349065 0.000000 1156.577299 105.965926 46.631061 54.930000 102.740000 -133.132996 0.000000


~/ipp/helpers/detrend_process.pl 1 850131b mc bias 850131b.fits 850131b
~/ipp/helpers/detrend_process.pl 1 850132b mc bias 850132b.fits 850132b
~/ipp/helpers/detrend_process.pl 1 850133b mc bias 850133b.fits 850133b
~/ipp/helpers/detrend_process.pl 1 850403b mc bias 850403b.fits 850403b
~/ipp/helpers/detrend_process.pl 1 850539b mc bias 850539b.fits 850539b
~/ipp/helpers/detrend_process.pl 1 850540b mc bias 850540b.fits 850540b
~/ipp/helpers/detrend_process.pl 1 850541b mc bias 850541b.fits 850541b

### Get list of imfiles to stack:

price@alala:/data/alala/price/megacam>dettool -tostack -simple
1 0 bias mc

### Stack the imfiles:

price@alala:/data/alala/price/megacam>~/ipp/helpers/detrend_stack.pl 1 0 mc bias

#### NB: dettool -addstacked wants statistics: need to add these into
####     ppMerge, and parse with detrend_stack.pl; DONE

### Create residuals from the stack

price@alala:/data/alala/price/megacam>dettool -toresid -simple
0 1 850131b mc o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.565582
0 1 850132b mc o_850132b.mc.fit PPIMAGE_O 1.748621 1.891902 292.655562
0 1 850133b mc o_850133b.mc.fit PPIMAGE_O 1.578467 2.000582 292.371550
0 1 850403b mc o_850403b.mc.fit PPIMAGE_O 1.297346 3.104167 293.029486
0 1 850539b mc o_850539b.mc.fit PPIMAGE_O 1.468040 2.212155 292.834501
0 1 850540b mc o_850540b.mc.fit PPIMAGE_O 1.403641 2.998480 292.843640
0 1 850541b mc o_850541b.mc.fit PPIMAGE_O 1.780084 1.854754 292.975948

### Need det_type, stacked (actually normalised) URI  ---> JH to provide; DONE

~/ipp/helpers/detrend_create_resid.pl 1 0 850131b mc bias bias_mc_1_0.fit o_850131b.mc.fit 850131b
~/ipp/helpers/detrend_create_resid.pl 1 0 850132b mc bias bias_mc_1_0.fit o_850132b.mc.fit 850132b
~/ipp/helpers/detrend_create_resid.pl 1 0 850133b mc bias bias_mc_1_0.fit o_850133b.mc.fit 850133b
~/ipp/helpers/detrend_create_resid.pl 1 0 850403b mc bias bias_mc_1_0.fit o_850403b.mc.fit 850403b
~/ipp/helpers/detrend_create_resid.pl 1 0 850539b mc bias bias_mc_1_0.fit o_850539b.mc.fit 850539b
~/ipp/helpers/detrend_create_resid.pl 1 0 850540b mc bias bias_mc_1_0.fit o_850540b.mc.fit 850540b
~/ipp/helpers/detrend_create_resid.pl 1 0 850541b mc bias bias_mc_1_0.fit o_850541b.mc.fit 850541b


### List of exposures that have been residualized:

price@alala:/data/alala/price/megacam>dettool -toresidexp -simple
1 0 bias 850131b T
1 0 bias 850132b T
1 0 bias 850133b T
1 0 bias 850403b T
1 0 bias 850539b T
1 0 bias 850540b T
1 0 bias 850541b T

### Check exposures:

~/ipp/helpers/detrend_reject_imfile.pl 1 0 850131b bias
~/ipp/helpers/detrend_reject_imfile.pl 1 0 850132b bias
~/ipp/helpers/detrend_reject_imfile.pl 1 0 850133b bias
~/ipp/helpers/detrend_reject_imfile.pl 1 0 850403b bias
~/ipp/helpers/detrend_reject_imfile.pl 1 0 850539b bias
~/ipp/helpers/detrend_reject_imfile.pl 1 0 850540b bias
~/ipp/helpers/detrend_reject_imfile.pl 1 0 850541b bias

### Need to allow ppImage to write a JPEG file from a pre-binned input.
### Tried setting BIN1.XBIN=1,BIN2.YBIN=1 and BIN1.JPEG=TRUE, but it doesn't seem to work.


### Get list of detrend runs to check for bad exposures:

price@alala:/data/alala/price/megacam>dettool -residdetrun -simple
1 0 bias

### Do the check:

~/ipp/helpers/detrend_reject_exp.pl 1 0 bias


### And that's it.

### Didn't try:
### (1) Iterating
### (2) Normalisation
### (3) JPEG creation --- need to use chip-mosaicked camera



###########################################################################################################

### Trying some flats


pxinject -newExp -exp_id 806237f -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 806238f -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 806239f -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1
pxinject -newExp -exp_id 806240f -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1

pxinject -newImfile -exp_id 806237f -class fpa -class_id mc -uri 806237f.fits
pxinject -newImfile -exp_id 806238f -class fpa -class_id mc -uri 806238f.fits
pxinject -newImfile -exp_id 806239f -class fpa -class_id mc -uri 806239f.fits
pxinject -newImfile -exp_id 806240f -class fpa -class_id mc -uri 806240f.fits


price@alala:/data/alala/price/megacam/flats>p0tool -pendingimfile -simple
806237f fpa mc 806237f.fits
806238f fpa mc 806238f.fits
806239f fpa mc 806239f.fits
806240f fpa mc 806240f.fits

### ===>

~/ipp/helpers/phase0imfile.pl 806237f mc 806237f.fits
~/ipp/helpers/phase0imfile.pl 806238f mc 806238f.fits
~/ipp/helpers/phase0imfile.pl 806239f mc 806239f.fits
~/ipp/helpers/phase0imfile.pl 806240f mc 806240f.fits


price@alala:/data/alala/price/megacam/flats>p0tool -pendingexp -simple
806237f MegaCam CFHT bias 1
806238f MegaCam CFHT bias 1
806239f MegaCam CFHT bias 1
806240f MegaCam CFHT bias 1

### ===>

~/ipp/helpers/phase0exp.pl 806237f
~/ipp/helpers/phase0exp.pl 806238f
~/ipp/helpers/phase0exp.pl 806239f
~/ipp/helpers/phase0exp.pl 806240f


### Changing camera configuration file to use the bias we got earlier:
   PPIMAGE.BIAS       INPUT bias_mc_1_0.fit               {CHIP.NAME}   {CHIP.NAME}  NONE         CHIP       IMAGE   

### Changing ppImage_obd recipe to turn off dark:
DARK		BOOL	FALSE		# Dark subtraction


### Start the detrend run:

price@alala:/data/alala/price/megacam/flats>dettool -definebyquery -det_type flat -camera mc
detRun MULTI #
 
detRun  METADATA
   det_id           S32       2
   iteration        S32       0
   det_type         STR       flat
END
 

price@alala:/data/alala/price/megacam/flats>dettool -raw -simple
2 flat 806237f fpa mc 806237f.fits FLAT g.MP9401 1.001000 0.341503 0.348775 99.112000 3687.433111 146.522771 74.441912 88.160000 275.250000 -133.033005 0.000000
2 flat 806238f fpa mc 806238f.fits FLAT g.MP9401 1.000000 0.383900 0.348775 99.121002 5208.304792 212.152572 89.617438 89.760000 310.990000 -133.033005 0.000000
2 flat 806239f fpa mc 806239f.fits FLAT g.MP9401 1.000000 0.395099 0.348775 80.097000 6311.806444 265.238880 100.970683 89.750000 310.150000 -133.033005 0.000000
2 flat 806240f fpa mc 806240f.fits FLAT g.MP9401 1.000000 0.405135 0.348775 60.098999 6957.866556 293.657405 108.000238 89.750000 309.930000 -133.033005 0.000000

### ===>


~/ipp/helpers/detrend_process.pl 2 806237f mc flat 806237f.fits 806237f
~/ipp/helpers/detrend_process.pl 2 806238f mc flat 806238f.fits 806238f
~/ipp/helpers/detrend_process.pl 2 806239f mc flat 806239f.fits 806239f
~/ipp/helpers/detrend_process.pl 2 806240f mc flat 806240f.fits 806240f

### Note: couple hundred psString leaks in ppImage here.


price@alala:/data/alala/price/megacam/flats>dettool -tostack -simple
2 0 flat mc

### ===>

price@alala:/data/alala/price/megacam>~/ipp/helpers/detrend_stack.pl 2 0 mc flat
