Index: /trunk/doc/ipptools/pipeline.txt =================================================================== --- /trunk/doc/ipptools/pipeline.txt (revision 8599) +++ /trunk/doc/ipptools/pipeline.txt (revision 8599) @@ -0,0 +1,233 @@ +Put Perl modules in ~/perl/ + +setenv PERL5LIB $HOME/perl/ + + +### Set up the tables + +price@alala:/home/panstarrs/price/ipp/ippTools/pXtools>pxadmin -create +price@alala:/home/panstarrs/price/ipp/ippTools/pXtools>pxadmin -recreate +*** delete the P2 tables? *** +*** to delete the tables, answer YES, and give password *** +*** WARNING: this action is permanent *** + +delete the P2 tables (YES/[n]): YES +enter dbh connection password: ipp + +### Inject everything into the DB + +pxinject -newExp -exp_id 850131b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850132b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850133b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850403b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850539b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850540b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850541b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850647b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850648b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850649b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850650b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850651b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850652b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850653b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850654b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850655b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850723b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850822b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850935b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 +pxinject -newExp -exp_id 850936b -inst MegaCam -telescope CFHT -exp_type bias -imfiles 1 + +pxinject -newImfile -exp_id 850131b -class fpa -class_id mc -uri 850131b.fits +pxinject -newImfile -exp_id 850132b -class fpa -class_id mc -uri 850132b.fits +pxinject -newImfile -exp_id 850133b -class fpa -class_id mc -uri 850133b.fits +pxinject -newImfile -exp_id 850403b -class fpa -class_id mc -uri 850403b.fits +pxinject -newImfile -exp_id 850539b -class fpa -class_id mc -uri 850539b.fits +pxinject -newImfile -exp_id 850540b -class fpa -class_id mc -uri 850540b.fits +pxinject -newImfile -exp_id 850541b -class fpa -class_id mc -uri 850541b.fits +pxinject -newImfile -exp_id 850647b -class fpa -class_id mc -uri 850647b.fits +pxinject -newImfile -exp_id 850648b -class fpa -class_id mc -uri 850648b.fits +pxinject -newImfile -exp_id 850649b -class fpa -class_id mc -uri 850649b.fits +pxinject -newImfile -exp_id 850650b -class fpa -class_id mc -uri 850650b.fits +pxinject -newImfile -exp_id 850651b -class fpa -class_id mc -uri 850651b.fits +pxinject -newImfile -exp_id 850652b -class fpa -class_id mc -uri 850652b.fits +pxinject -newImfile -exp_id 850653b -class fpa -class_id mc -uri 850653b.fits +pxinject -newImfile -exp_id 850654b -class fpa -class_id mc -uri 850654b.fits +pxinject -newImfile -exp_id 850655b -class fpa -class_id mc -uri 850655b.fits +pxinject -newImfile -exp_id 850723b -class fpa -class_id mc -uri 850723b.fits +pxinject -newImfile -exp_id 850822b -class fpa -class_id mc -uri 850822b.fits +pxinject -newImfile -exp_id 850935b -class fpa -class_id mc -uri 850935b.fits +pxinject -newImfile -exp_id 850936b -class fpa -class_id mc -uri 850936b.fits + +### Look for phase 0 imfiles: + +price@alala:/data/alala/price/megacam>p0tool -pendingimfile -simple +850131b fpa mc 850131b.fits +850132b fpa mc 850132b.fits +850133b fpa mc 850133b.fits +850403b fpa mc 850403b.fits +850539b fpa mc 850539b.fits +850540b fpa mc 850540b.fits +850541b fpa mc 850541b.fits +850647b fpa mc 850647b.fits +850648b fpa mc 850648b.fits +850649b fpa mc 850649b.fits +850650b fpa mc 850650b.fits +850651b fpa mc 850651b.fits +850652b fpa mc 850652b.fits +850653b fpa mc 850653b.fits +850654b fpa mc 850654b.fits +850655b fpa mc 850655b.fits +850723b fpa mc 850723b.fits +850822b fpa mc 850822b.fits +850935b fpa mc 850935b.fits +850936b fpa mc 850936b.fits + +### Run the phase 0 imfile script: + +~/ipp/helpers/phase0imfile.pl 850131b mc 850131b.fits +~/ipp/helpers/phase0imfile.pl 850132b mc 850132b.fits +~/ipp/helpers/phase0imfile.pl 850133b mc 850133b.fits +~/ipp/helpers/phase0imfile.pl 850403b mc 850403b.fits +~/ipp/helpers/phase0imfile.pl 850539b mc 850539b.fits +~/ipp/helpers/phase0imfile.pl 850540b mc 850540b.fits +~/ipp/helpers/phase0imfile.pl 850541b mc 850541b.fits +~/ipp/helpers/phase0imfile.pl 850647b mc 850647b.fits +~/ipp/helpers/phase0imfile.pl 850648b mc 850648b.fits +~/ipp/helpers/phase0imfile.pl 850649b mc 850649b.fits +~/ipp/helpers/phase0imfile.pl 850650b mc 850650b.fits +~/ipp/helpers/phase0imfile.pl 850651b mc 850651b.fits +~/ipp/helpers/phase0imfile.pl 850652b mc 850652b.fits +~/ipp/helpers/phase0imfile.pl 850653b mc 850653b.fits +~/ipp/helpers/phase0imfile.pl 850654b mc 850654b.fits +~/ipp/helpers/phase0imfile.pl 850655b mc 850655b.fits +~/ipp/helpers/phase0imfile.pl 850723b mc 850723b.fits +~/ipp/helpers/phase0imfile.pl 850822b mc 850822b.fits +~/ipp/helpers/phase0imfile.pl 850935b mc 850935b.fits +~/ipp/helpers/phase0imfile.pl 850936b mc 850936b.fits + +### Look for phase 0 exposures: + +price@alala:/data/alala/price/megacam>p0tool -pendingexp -simple +850131b MegaCam CFHT bias 1 +850132b MegaCam CFHT bias 1 +850133b MegaCam CFHT bias 1 +850403b MegaCam CFHT bias 1 +850539b MegaCam CFHT bias 1 +850540b MegaCam CFHT bias 1 +850541b MegaCam CFHT bias 1 +850647b MegaCam CFHT bias 1 +850648b MegaCam CFHT bias 1 +850649b MegaCam CFHT bias 1 +850650b MegaCam CFHT bias 1 +850651b MegaCam CFHT bias 1 +850652b MegaCam CFHT bias 1 +850653b MegaCam CFHT bias 1 +850654b MegaCam CFHT bias 1 +850655b MegaCam CFHT bias 1 +850723b MegaCam CFHT bias 1 +850822b MegaCam CFHT bias 1 +850935b MegaCam CFHT bias 1 +850936b MegaCam CFHT bias 1 + + +### Run the phase 0 exposure script: + +~/ipp/helpers/phase0exp.pl 850131b +~/ipp/helpers/phase0exp.pl 850132b +~/ipp/helpers/phase0exp.pl 850133b +~/ipp/helpers/phase0exp.pl 850403b +~/ipp/helpers/phase0exp.pl 850539b +~/ipp/helpers/phase0exp.pl 850540b +~/ipp/helpers/phase0exp.pl 850541b +~/ipp/helpers/phase0exp.pl 850647b +~/ipp/helpers/phase0exp.pl 850648b +~/ipp/helpers/phase0exp.pl 850649b +~/ipp/helpers/phase0exp.pl 850650b +~/ipp/helpers/phase0exp.pl 850651b +~/ipp/helpers/phase0exp.pl 850652b +~/ipp/helpers/phase0exp.pl 850653b +~/ipp/helpers/phase0exp.pl 850654b +~/ipp/helpers/phase0exp.pl 850655b +~/ipp/helpers/phase0exp.pl 850723b +~/ipp/helpers/phase0exp.pl 850822b +~/ipp/helpers/phase0exp.pl 850935b +~/ipp/helpers/phase0exp.pl 850936b + +### Start a bias detrend run + +price@alala:/data/alala/price/megacam>dettool -definebyquery -det_type bias -camera mc + +detRun MULTI # + +detRun METADATA + det_id S32 1 + iteration S32 0 + det_type STR bias +END + + + +### Get list of pending raw images: + + +price@alala:/data/alala/price/megacam>dettool -raw -simple + psMetadataRemove (psMetadata.c:713) : Failed to remove metadata item, position, from metadata table. +1 bias 850131b fpa mc 850131b.fits BIAS r.MP9601 1.656000 5.496118 0.349071 0.000000 1155.934692 107.037283 46.214947 37.160000 105.480000 -133.067001 0.000000 +1 bias 850132b fpa mc 850132b.fits BIAS r.MP9601 1.656000 5.499327 0.349071 0.000000 1155.431957 107.464479 46.207451 37.160000 105.480000 -133.067001 0.000000 +1 bias 850133b fpa mc 850133b.fits BIAS r.MP9601 1.656000 5.502535 0.349071 0.000000 1154.871040 107.485792 46.227965 37.160000 105.480000 -133.067001 0.000000 +1 bias 850403b fpa mc 850403b.fits BIAS r.MP9601 1.000000 2.184885 0.349065 0.000000 1156.454996 105.813700 46.699662 89.590000 272.850000 -133.100006 0.000000 +1 bias 850539b fpa mc 850539b.fits BIAS HaOFF.MP7604 1.000000 2.928339 0.349071 0.000000 1155.991272 105.710902 46.830444 -9999.900000 -9999.900000 -133.100006 0.000000 +1 bias 850540b fpa mc 850540b.fits BIAS HaOFF.MP7604 1.000000 2.934318 0.349071 0.000000 1156.668537 106.228856 46.638641 -9999.900000 -9999.900000 -133.132996 0.000000 +1 bias 850541b fpa mc 850541b.fits BIAS NONE 1.035000 1.539753 0.349058 0.000000 1157.512387 105.972075 46.692062 75.000000 272.590000 -133.233002 0.000000 +1 bias 850647b fpa mc 850647b.fits BIAS i.MP9701 1.000000 5.661125 0.349065 0.000000 1159.236140 105.088919 46.581451 89.640000 273.780000 -133.132996 0.000000 +1 bias 850648b fpa mc 850648b.fits BIAS i.MP9701 1.000000 5.664260 0.349065 0.000000 1159.642160 105.172955 46.462718 89.640000 273.780000 -133.132996 0.000000 +1 bias 850649b fpa mc 850649b.fits BIAS i.MP9701 1.000000 5.667395 0.349065 0.000000 1159.877803 105.578676 46.323573 89.640000 273.780000 -133.132996 0.000000 +1 bias 850650b fpa mc 850650b.fits BIAS i.MP9701 1.000000 5.670604 0.349065 0.000000 1159.296260 105.885271 46.306679 89.640000 273.780000 -133.132996 0.000000 +1 bias 850651b fpa mc 850651b.fits BIAS i.MP9701 1.000000 5.673812 0.349065 0.000000 1159.130751 106.263144 46.198493 89.640000 273.780000 -133.132996 0.000000 +1 bias 850652b fpa mc 850652b.fits BIAS i.MP9701 1.000000 5.677020 0.349065 0.000000 1158.711761 106.354676 46.160953 89.640000 273.780000 -133.100006 0.000000 +1 bias 850653b fpa mc 850653b.fits BIAS i.MP9701 1.000000 5.680301 0.349065 0.000000 1157.834599 106.646493 46.197773 89.640000 273.780000 -133.100006 0.000000 +1 bias 850654b fpa mc 850654b.fits BIAS i.MP9701 1.000000 5.683510 0.349065 0.000000 1157.518896 106.736919 46.124295 89.640000 273.780000 -133.067001 0.000000 +1 bias 850655b fpa mc 850655b.fits BIAS i.MP9701 1.000000 5.686791 0.349065 0.000000 1156.908897 107.101147 46.126100 89.640000 273.780000 -133.167007 0.000000 +1 bias 850723b fpa mc 850723b.fits BIAS i.MP9701 1.000000 1.681629 0.349065 0.000000 1156.671385 105.943337 46.696208 89.640000 273.780000 -133.167007 0.000000 +1 bias 850822b fpa mc 850822b.fits BIAS r.MP9601 1.318000 1.317600 0.349039 0.000000 1156.904900 105.906221 46.700064 49.330000 156.180000 -133.167007 0.000000 +1 bias 850935b fpa mc 850935b.fits BIAS i.MP9701 1.222000 0.670460 0.349065 0.000000 1156.704276 105.995095 46.740883 54.930000 102.740000 -133.167007 0.000000 +1 bias 850936b fpa mc 850936b.fits BIAS r.MP9601 1.222000 2.741378 0.349065 0.000000 1156.577299 105.965926 46.631061 54.930000 102.740000 -133.132996 0.000000 + + +~/ipp/helpers/detrend_process.pl 1 850131b mc bias 850131b.fits 850131b +~/ipp/helpers/detrend_process.pl 1 850132b mc bias 850132b.fits 850132b +~/ipp/helpers/detrend_process.pl 1 850133b mc bias 850133b.fits 850133b +~/ipp/helpers/detrend_process.pl 1 850403b mc bias 850403b.fits 850403b +~/ipp/helpers/detrend_process.pl 1 850539b mc bias 850539b.fits 850539b +~/ipp/helpers/detrend_process.pl 1 850540b mc bias 850540b.fits 850540b +~/ipp/helpers/detrend_process.pl 1 850541b mc bias 850541b.fits 850541b + +### Get list of imfiles to stack: + +price@alala:/data/alala/price/megacam>dettool -tostack -simple +1 0 bias mc + +### Stack the imfiles: + +price@alala:/data/alala/price/megacam>~/ipp/helpers/detrend_stack.pl 1 0 mc bias + +#### NB: dettool -addstacked wants statistics: need to add these into ppMerge, and parse with detrend_stack.pl + +price@alala:/data/alala/price/megacam>dettool -toresid -simple +0 1 850131b mc o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.565582 +0 1 850132b mc o_850132b.mc.fit PPIMAGE_O 1.748621 1.891902 292.655562 +0 1 850133b mc o_850133b.mc.fit PPIMAGE_O 1.578467 2.000582 292.371550 +0 1 850403b mc o_850403b.mc.fit PPIMAGE_O 1.297346 3.104167 293.029486 +0 1 850539b mc o_850539b.mc.fit PPIMAGE_O 1.468040 2.212155 292.834501 +0 1 850540b mc o_850540b.mc.fit PPIMAGE_O 1.403641 2.998480 292.843640 +0 1 850541b mc o_850541b.mc.fit PPIMAGE_O 1.780084 1.854754 292.975948 + +### Need det_type, stacked (actually normalised) URI ---> JH to provide + +~/ipp/helpers/detrend_create_resid.pl 1 0 850131b mc bias bias_mc_1_0.fit o_850131b.mc.fit 850131b +~/ipp/helpers/detrend_create_resid.pl 1 0 850132b mc bias bias_mc_1_0.fit o_850132b.mc.fit 850132b +~/ipp/helpers/detrend_create_resid.pl 1 0 850133b mc bias bias_mc_1_0.fit o_850133b.mc.fit 850133b +~/ipp/helpers/detrend_create_resid.pl 1 0 850403b mc bias bias_mc_1_0.fit o_850403b.mc.fit 850403b +~/ipp/helpers/detrend_create_resid.pl 1 0 850539b mc bias bias_mc_1_0.fit o_850539b.mc.fit 850539b +~/ipp/helpers/detrend_create_resid.pl 1 0 850540b mc bias bias_mc_1_0.fit o_850540b.mc.fit 850540b +~/ipp/helpers/detrend_create_resid.pl 1 0 850541b mc bias bias_mc_1_0.fit o_850541b.mc.fit 850541b