Index: /branches/cleanup/ippTools/share/dettool_tonormalizedstat.sql =================================================================== --- /branches/cleanup/ippTools/share/dettool_tonormalizedstat.sql (revision 18152) +++ /branches/cleanup/ippTools/share/dettool_tonormalizedstat.sql (revision 18153) @@ -16,5 +16,6 @@ iteration, camera, - workdir + workdir, + class_id FROM (SELECT DISTINCT Index: /branches/cleanup/ippTools/share/dettool_toresidimfile.sql =================================================================== --- /branches/cleanup/ippTools/share/dettool_toresidimfile.sql (revision 18152) +++ /branches/cleanup/ippTools/share/dettool_toresidimfile.sql (revision 18153) @@ -77,2 +77,3 @@ AND detResidImfile.exp_id IS NULL AND detResidImfile.class_id IS NULL +ORDER BY exp_id Index: /branches/cleanup/ippTools/share/faketool_pendingimfile.sql =================================================================== --- /branches/cleanup/ippTools/share/faketool_pendingimfile.sql (revision 18152) +++ /branches/cleanup/ippTools/share/faketool_pendingimfile.sql (revision 18153) @@ -24,4 +24,5 @@ camProcessedExp.path_base as cam_path_base, rawExp.exp_name, + rawExp.exp_tag, rawExp.camera, rawExp.telescope, Index: /branches/cleanup/ippTools/share/warptool_imfile.sql =================================================================== --- /branches/cleanup/ippTools/share/warptool_imfile.sql (revision 18152) +++ /branches/cleanup/ippTools/share/warptool_imfile.sql (revision 18153) @@ -2,4 +2,5 @@ rawImfile.*, warpRun.fake_id, + camRun.cam_id as cam_id, chipProcessedImfile.uri as chip_uri, chipProcessedImfile.path_base as chip_path_base, Index: /branches/cleanup/ippTools/src/camtool.c =================================================================== --- /branches/cleanup/ippTools/src/camtool.c (revision 18152) +++ /branches/cleanup/ippTools/src/camtool.c (revision 18153) @@ -30,5 +30,5 @@ #include "camtool.h" -static bool queueMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool pendingexpMode(pxConfig *config); @@ -60,5 +60,5 @@ switch (config->mode) { - MODECASE(CAMTOOL_MODE_QUEUE, queueMode); + MODECASE(CAMTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); MODECASE(CAMTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(CAMTOOL_MODE_PENDINGEXP, pendingexpMode); @@ -93,5 +93,5 @@ -static bool queueMode(pxConfig *config) +static bool definebyqueryMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); Index: /branches/cleanup/ippTools/src/camtool.h =================================================================== --- /branches/cleanup/ippTools/src/camtool.h (revision 18152) +++ /branches/cleanup/ippTools/src/camtool.h (revision 18153) @@ -25,5 +25,5 @@ typedef enum { CAMTOOL_MODE_NONE = 0x0, - CAMTOOL_MODE_QUEUE, + CAMTOOL_MODE_DEFINEBYQUERY, CAMTOOL_MODE_UPDATERUN, CAMTOOL_MODE_PENDINGEXP, Index: /branches/cleanup/ippTools/src/camtoolConfig.c =================================================================== --- /branches/cleanup/ippTools/src/camtoolConfig.c (revision 18152) +++ /branches/cleanup/ippTools/src/camtoolConfig.c (revision 18153) @@ -46,101 +46,101 @@ } - // -queue - psMetadata *queueArgs = psMetadataAlloc(); + // -definebyquery + psMetadata *definebydefinebyqueryArgs = psMetadataAlloc(); // XXX need to allow multiple chip_ids - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-chip_id", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", NULL); // XXX need to allow multiple exp_ids - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_id", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_name", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-inst", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-telescope", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); - psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-dateobs_begin", 0, + psMetadataAddTime(definebydefinebyqueryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); - psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-dateobs_end", 0, + psMetadataAddTime(definebydefinebyqueryArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_tag", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_type", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-filelevel", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-reduction", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-filter", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-airmass_min", 0, + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-airmass_max", 0, + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra_min", 0, - "define min", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra_max", 0, - "define max", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl_min", 0, - "define min", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl_max", 0, - "define max", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-exp_time_min", 0, - "define min", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-exp_time_max", 0, - "define max", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-ra_min", 0, + "define min", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-ra_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-decl_min", 0, + "define min", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-decl_max", 0, + "define max", NAN); + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min", NAN); + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max", NAN); + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_min", 0, - "define max", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_max", 0, - "define max", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, - "define max", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, - "define max", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, - "define max", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, - "define max", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-alt_min", 0, - "define min", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-alt_max", 0, - "define max", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-az_min", 0, - "define min", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-az_max", 0, - "define max", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_min", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-alt_min", 0, + "define min", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-alt_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-az_min", 0, + "define min", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-az_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-posang_min", 0, + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-posang_max", 0, + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-object", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-solang_min", 0, + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-solang_max", 0, + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_workdir", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_label", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_reduction", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_expgroup", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_dvodb", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_tess_id", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tess ID", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_end_stage", 0, + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); - psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-all", 0, + psMetadataAddBool(definebydefinebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); @@ -462,5 +462,5 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-queue", "", CAMTOOL_MODE_QUEUE, queueArgs); + PXOPT_ADD_MODE("-definebyquery", "", CAMTOOL_MODE_DEFINEBYQUERY, definebydefinebyqueryArgs); PXOPT_ADD_MODE("-updaterun", "", CAMTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-pendingexp", "", CAMTOOL_MODE_PENDINGEXP, pendingexpArgs); Index: /branches/cleanup/ippTools/src/chiptool.c =================================================================== --- /branches/cleanup/ippTools/src/chiptool.c (revision 18152) +++ /branches/cleanup/ippTools/src/chiptool.c (revision 18153) @@ -34,5 +34,5 @@ #include "camtool.h" -static bool queueMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool pendingimfileMode(pxConfig *config); @@ -68,5 +68,5 @@ switch (config->mode) { - MODECASE(CHIPTOOL_MODE_QUEUE, queueMode); + MODECASE(CHIPTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); MODECASE(CHIPTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(CHIPTOOL_MODE_PENDINGIMFILE, pendingimfileMode); @@ -104,5 +104,5 @@ -static bool queueMode(pxConfig *config) +static bool definebyqueryMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); Index: /branches/cleanup/ippTools/src/chiptool.h =================================================================== --- /branches/cleanup/ippTools/src/chiptool.h (revision 18152) +++ /branches/cleanup/ippTools/src/chiptool.h (revision 18153) @@ -25,5 +25,5 @@ typedef enum { CHIPTOOL_MODE_NONE = 0x0, - CHIPTOOL_MODE_QUEUE, + CHIPTOOL_MODE_DEFINEBYQUERY, CHIPTOOL_MODE_UPDATERUN, CHIPTOOL_MODE_PENDINGIMFILE, Index: /branches/cleanup/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/cleanup/ippTools/src/chiptoolConfig.c (revision 18152) +++ /branches/cleanup/ippTools/src/chiptoolConfig.c (revision 18153) @@ -44,5 +44,5 @@ } - // -queue + // -definebyquery psMetadata *queueArgs = psMetadataAlloc(); // XXX need to allow multiple exp_ids @@ -293,6 +293,6 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-queue", "create runs from raw exposures", - CHIPTOOL_MODE_QUEUE, queueArgs); + PXOPT_ADD_MODE("-definebyquery", "create runs from raw exposures", + CHIPTOOL_MODE_DEFINEBYQUERY, queueArgs); PXOPT_ADD_MODE("-updaterun", "change chip run properties", CHIPTOOL_MODE_UPDATERUN, updaterunArgs); Index: /branches/cleanup/ippTools/src/dettool.c =================================================================== --- /branches/cleanup/ippTools/src/dettool.c (revision 18152) +++ /branches/cleanup/ippTools/src/dettool.c (revision 18153) @@ -535,4 +535,7 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(randomSubset, config->args, "-random_subset", false); + PXOPT_LOOKUP_S32(randomLimit, config->args, "-random_limit", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); PXOPT_LOOKUP_TIME(time_begin, config->args, "-time_begin", false, false); @@ -549,5 +552,5 @@ PXOPT_COPY_STR(config->args, where, "-select_uri", "uri", "=="); PXOPT_COPY_TIME(config->args, where, "-select_dateobs_begin", "dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-select_dateobs_end", "dateobs", "<"); + PXOPT_COPY_TIME(config->args, where, "-select_dateobs_end", "dateobs", "<="); PXOPT_COPY_F32(config->args, where, "-select_airmass_min", "airmass", ">="); PXOPT_COPY_F32(config->args, where, "-select_airmass_max", "airmass", "<="); @@ -557,4 +560,6 @@ PXOPT_COPY_F32(config->args, where, "-select_ccd_temp_min", "ccd_temp", ">="); PXOPT_COPY_F32(config->args, where, "-select_ccd_temp_max", "ccd_temp", "<="); + PXOPT_COPY_F32(config->args, where, "-select_pon_time_min", "pon_time", ">="); + PXOPT_COPY_F32(config->args, where, "-select_pon_time_max", "pon_time", "<="); PXOPT_COPY_F32(config->args, where, "-select_posang_min", "posang", ">="); PXOPT_COPY_F32(config->args, where, "-select_posang_max", "posang", "<="); @@ -584,4 +589,23 @@ psFree(detrendExps); return true; + } + + if (randomSubset && (randomLimit < detrendExps->n)) { + // generate a random-valued vector, return an index sorted by the random values + psVector *randomVector = psVectorAlloc(detrendExps->n, PS_TYPE_F32); // random values + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + for (int i = 0; i < randomVector->n; i++) { + randomVector->data.F32[i] = psRandomUniform(rng); + } + psVector *indexVector = psVectorSortIndex(NULL, randomVector); + + // accept for first n of the sequence + psArray *subset = psArrayAlloc (randomLimit); + for (int i = 0; i < randomLimit; i++ ){ + int j = indexVector->data.S32[i]; + subset->data[i] = psMemIncrRefCounter (detrendExps->data[j]); + } + psFree (detrendExps); + detrendExps = subset; } Index: /branches/cleanup/ippTools/src/dettoolConfig.c =================================================================== --- /branches/cleanup/ippTools/src/dettoolConfig.c (revision 18152) +++ /branches/cleanup/ippTools/src/dettoolConfig.c (revision 18153) @@ -194,4 +194,8 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_pon_time_min", 0, + "define min power-on time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_pon_time_max", 0, + "define max power-on time", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); @@ -210,4 +214,9 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-random_subset", 0, + "use a random subset of elements", false); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random_limit", 0, + "use this number of random elements", 20); // -definebydetrun Index: /branches/cleanup/ippTools/src/difftool.c =================================================================== --- /branches/cleanup/ippTools/src/difftool.c (revision 18152) +++ /branches/cleanup/ippTools/src/difftool.c (revision 18153) @@ -40,5 +40,5 @@ static bool revertdiffskyfileMode(pxConfig *config); static bool definepoprunMode(pxConfig *config); -static bool queueMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); static bool setdiffRunState(pxConfig *config, const char *diff_id, const char *state); @@ -71,5 +71,5 @@ MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileMode); MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunMode); - MODECASE(DIFFTOOL_MODE_QUEUE, queueMode); + MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); default: psAbort("invalid option (this should not happen)"); @@ -802,5 +802,5 @@ -static bool queueMode(pxConfig *config) +static bool definebyqueryMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); Index: /branches/cleanup/ippTools/src/difftool.h =================================================================== --- /branches/cleanup/ippTools/src/difftool.h (revision 18152) +++ /branches/cleanup/ippTools/src/difftool.h (revision 18153) @@ -34,5 +34,5 @@ DIFFTOOL_MODE_REVERTDIFFSKYFILE, DIFFTOOL_MODE_DEFINEPOPRUN, - DIFFTOOL_MODE_QUEUE, + DIFFTOOL_MODE_DEFINEBYQUERY, } difftoolMode; Index: /branches/cleanup/ippTools/src/difftoolConfig.c =================================================================== --- /branches/cleanup/ippTools/src/difftoolConfig.c (revision 18152) +++ /branches/cleanup/ippTools/src/difftoolConfig.c (revision 18153) @@ -180,5 +180,5 @@ "define stack ID for input", NULL); - // -queue + // -definebyquery psMetadata *queueArgs = psMetadataAlloc(); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-workdir", 0, @@ -203,5 +203,5 @@ PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs); PXOPT_ADD_MODE("-definepoprun", "", DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunArgs); - PXOPT_ADD_MODE("-queue", "", DIFFTOOL_MODE_QUEUE, queueArgs); + PXOPT_ADD_MODE("-definebyquery", "", DIFFTOOL_MODE_DEFINEBYQUERY, queueArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /branches/cleanup/ippTools/src/faketool.c =================================================================== --- /branches/cleanup/ippTools/src/faketool.c (revision 18152) +++ /branches/cleanup/ippTools/src/faketool.c (revision 18153) @@ -33,5 +33,5 @@ #include "faketool.h" -static bool queueMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool pendingexpMode(pxConfig *config); @@ -65,5 +65,5 @@ switch (config->mode) { - MODECASE(FAKETOOL_MODE_QUEUE, queueMode); + MODECASE(FAKETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); MODECASE(FAKETOOL_MODE_UPDATERUN, updaterunMode); MODECASE(FAKETOOL_MODE_PENDINGEXP, pendingexpMode); @@ -99,5 +99,5 @@ -static bool queueMode(pxConfig *config) +static bool definebyqueryMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); Index: /branches/cleanup/ippTools/src/faketool.h =================================================================== --- /branches/cleanup/ippTools/src/faketool.h (revision 18152) +++ /branches/cleanup/ippTools/src/faketool.h (revision 18153) @@ -25,5 +25,5 @@ typedef enum { FAKETOOL_MODE_NONE = 0x0, - FAKETOOL_MODE_QUEUE, + FAKETOOL_MODE_DEFINEBYQUERY, FAKETOOL_MODE_UPDATERUN, FAKETOOL_MODE_PENDINGIMFILE, Index: /branches/cleanup/ippTools/src/faketoolConfig.c =================================================================== --- /branches/cleanup/ippTools/src/faketoolConfig.c (revision 18152) +++ /branches/cleanup/ippTools/src/faketoolConfig.c (revision 18153) @@ -44,5 +44,5 @@ } - // -queue + // -definebyquery psMetadata *queueArgs = psMetadataAlloc(); // XXX need to allow multiple exp_ids @@ -426,6 +426,6 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-queue", "create runs from raw exposures", - FAKETOOL_MODE_QUEUE, queueArgs); + PXOPT_ADD_MODE("-definebyquery", "create runs from raw exposures", + FAKETOOL_MODE_DEFINEBYQUERY, queueArgs); PXOPT_ADD_MODE("-updaterun", "change fake run properties", FAKETOOL_MODE_UPDATERUN, updaterunArgs); Index: /branches/cleanup/ippTools/src/magictool.c =================================================================== --- /branches/cleanup/ippTools/src/magictool.c (revision 18152) +++ /branches/cleanup/ippTools/src/magictool.c (revision 18153) @@ -31,5 +31,5 @@ #include "magictool.h" -static bool queueMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); static psS64 definerunMode(pxConfig *config); static bool updaterunMode(pxConfig *config); @@ -65,5 +65,5 @@ switch (config->mode) { - MODECASE(MAGICTOOL_MODE_QUEUE, queueMode); + MODECASE(MAGICTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); MODECASE(MAGICTOOL_MODE_DEFINERUN, definerunMode); MODECASE(MAGICTOOL_MODE_UPDATERUN, updaterunMode); @@ -98,5 +98,5 @@ } -static bool queueMode(pxConfig *config) +static bool definebyqueryMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); Index: /branches/cleanup/ippTools/src/magictool.h =================================================================== --- /branches/cleanup/ippTools/src/magictool.h (revision 18152) +++ /branches/cleanup/ippTools/src/magictool.h (revision 18153) @@ -25,5 +25,5 @@ typedef enum { MAGICTOOL_MODE_NONE = 0x0, - MAGICTOOL_MODE_QUEUE, + MAGICTOOL_MODE_DEFINEBYQUERY, MAGICTOOL_MODE_DEFINERUN, MAGICTOOL_MODE_UPDATERUN, Index: /branches/cleanup/ippTools/src/magictoolConfig.c =================================================================== --- /branches/cleanup/ippTools/src/magictoolConfig.c (revision 18152) +++ /branches/cleanup/ippTools/src/magictoolConfig.c (revision 18153) @@ -47,5 +47,5 @@ psTime *now = psTimeGetNow(PS_TIME_TAI); - // -queue + // -definebyquery psMetadata *queueArgs = psMetadataAlloc(); @@ -160,5 +160,5 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-queue", "", MAGICTOOL_MODE_QUEUE, queueArgs); + PXOPT_ADD_MODE("-definebyquery", "", MAGICTOOL_MODE_DEFINEBYQUERY, queueArgs); PXOPT_ADD_MODE("-definerun", "", MAGICTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "", MAGICTOOL_MODE_UPDATERUN, updaterunArgs); Index: /branches/cleanup/ippTools/src/stacktool.c =================================================================== --- /branches/cleanup/ippTools/src/stacktool.c (revision 18152) +++ /branches/cleanup/ippTools/src/stacktool.c (revision 18153) @@ -31,5 +31,5 @@ #include "stacktool.h" -static bool queueMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); static bool definerunMode(pxConfig *config); static bool updaterunMode(pxConfig *config); @@ -61,5 +61,5 @@ switch (config->mode) { - MODECASE(STACKTOOL_MODE_QUEUE, queueMode); + MODECASE(STACKTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); MODECASE(STACKTOOL_MODE_DEFINERUN, definerunMode); MODECASE(STACKTOOL_MODE_UPDATERUN, updaterunMode); @@ -92,5 +92,5 @@ -static bool queueMode(pxConfig *config) +static bool definebyqueryMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -107,5 +107,5 @@ PXOPT_COPY_STR(config->args, whereRE, "-select_uri", "uri", "=="); PXOPT_COPY_TIME(config->args, whereRE, "-select_dateobs_begin", "dateobs", ">="); - PXOPT_COPY_TIME(config->args, whereRE, "-select_dateobs_end", "dateobs", "<"); + PXOPT_COPY_TIME(config->args, whereRE, "-select_dateobs_end", "dateobs", "<="); PXOPT_COPY_F32(config->args, whereRE, "-select_airmass_min", "airmass", ">="); PXOPT_COPY_F32(config->args, whereRE, "-select_airmass_max", "airmass", "<="); @@ -120,4 +120,6 @@ PXOPT_COPY_F32(config->args, whereRE, "-select_solang_max", "solang", "<="); PXOPT_COPY_STR(config->args, whereWSF, "-select_skycell_id", "skycell_id", "=="); + PXOPT_LOOKUP_BOOL(randomSubset, config->args, "-random_subset", false); + PXOPT_LOOKUP_S32(randomLimit, config->args, "-random_limit", false, false); if (!psListLength(whereRE->list) && !psListLength(whereWSF->list) && @@ -182,15 +184,77 @@ } + // valid_warpids should be NULL unless randomSubset is specified + psVector *valid_warpids = NULL; + if (randomSubset && (randomLimit < psArrayLength(output))) { + // loop over the array of metadata and figure out which warp_id are in + // the result set + psArray *warps = psArrayAlloc(0); + for (long i = 0; i < psArrayLength(output); i++) { +// psMetadata *md = output->data[i]; + + // pull out the warp_id + bool status = false; +// psS64 warp_id = psMetadataLookupS64(&status, md, "warp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup warp_id"); + psFree(output); + psFree(warps); + return NULL; + } + +// psArrayAdd(warps, 0, warp_id); + } + + // generate a random-valued vector, return an index sorted by the + // random values + psVector *randomVector = psVectorAlloc(psArrayLength(output), PS_TYPE_F32); + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + for (int i = 0; i < randomVector->n; i++) { + randomVector->data.F32[i] = psRandomUniform(rng); + } + psVector *indexVector = psVectorSortIndex(NULL, randomVector); + psFree(randomVector); + + // accept for first n of the sequence + valid_warpids = psVectorAlloc(randomLimit, PS_TYPE_S64); + for (int i = 0; i < randomLimit; i++ ){ +// int j = indexVector->data.S32[i]; +// valid_warpids->data[i] = psMemIncrRefCounter (warps->data[j]); + } + psFree(indexVector); + psFree(warps); + } + + psHash *stacks = psHashAlloc(psArrayLength(output)); - // loop over the array of metadata + // loop over the array of metadata and build a hash keyed by tess_id & + // skycell_id for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; - // pull out the warp_id, skycell_id, & tess_id bool status; + if (valid_warpids) { +// psS64 warp_id = psMetadataLookupS64(&status, md, "warp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup warp_id"); + psFree(valid_warpids); + psFree(output); + return NULL; + } + + // is this a warp_id that should be left out of the stack set(s)? +// for (long i = 0; i < psArrayLength(valid_warpids); i++) { +// if (valid_warpids->data[i] == warp_id) { +// continue; +// } +// } + } + + // pull out the skycell_id, & tess_id psString skycell_id = psMetadataLookupStr(&status, md, "skycell_id"); if (!status) { psError(PS_ERR_UNKNOWN, false, "failed to lookup skycell_id"); + psFree(valid_warpids); psFree(output); return NULL; @@ -200,4 +264,5 @@ if (!status) { psError(PS_ERR_UNKNOWN, false, "failed to lookup tess_id"); + psFree(valid_warpids); psFree(output); return NULL; @@ -220,4 +285,5 @@ } + psFree(valid_warpids); psFree(output); Index: /branches/cleanup/ippTools/src/stacktool.h =================================================================== --- /branches/cleanup/ippTools/src/stacktool.h (revision 18152) +++ /branches/cleanup/ippTools/src/stacktool.h (revision 18153) @@ -25,5 +25,5 @@ typedef enum { STACKTOOL_MODE_NONE = 0x0, - STACKTOOL_MODE_QUEUE, + STACKTOOL_MODE_DEFINEBYQUERY, STACKTOOL_MODE_DEFINERUN, STACKTOOL_MODE_UPDATERUN, Index: /branches/cleanup/ippTools/src/stacktoolConfig.c =================================================================== --- /branches/cleanup/ippTools/src/stacktoolConfig.c (revision 18152) +++ /branches/cleanup/ippTools/src/stacktoolConfig.c (revision 18153) @@ -45,54 +45,58 @@ psTime *now = psTimeGetNow(PS_TIME_TAI); - // -queue - psMetadata *queueArgs = psMetadataAlloc(); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-workdir", 0, + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-registered", 0, + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-select_skycell_id", 0, + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-select_exp_type", 0, + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_type", 0, "search for exp_type", "object"); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-select_inst", 0, + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_inst", 0, "search for camera", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-select_telescope", 0, + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_telescope", 0, "search for telescope", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-select_filter", 0, + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", 0, "search for filter", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-select_uri", 0, + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_uri", 0, "search for uri", NULL); - psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-select_dateobs_begin", 0, + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_begin", 0, "search for exposures by time (>=)", NULL); - psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-select_dateobs_end", 0, + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_end", 0, "search for exposures by time (<)", NULL); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-select_airmass_min", 0, + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_min", 0, "define min airmass", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-select_airmass_max", 0, + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_max", 0, "define max airmass", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max", 0, + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-select_exp_time_min", 0, + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_min", 0, "define min exposure time", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-select_exp_time_max", 0, + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_max", 0, "define max exposure time", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-select_ccd_temp_min", 0, + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_min", 0, "define min ccd tempature", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-select_ccd_temp_max", 0, + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_max", 0, "define max ccd tempature", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-select_posang_min", 0, + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-select_posang_max", 0, + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-select_solang_min", 0, + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_min", 0, "define min solar angle", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-select_solang_max", 0, + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_max", 0, "define max solar angle", NAN); - - psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-all", 0, + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-random_subset", 0, + "use a random subset of elements", false); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random_limit", 0, + "use this number of random elements", 20); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); - psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-simple", 0, + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -200,6 +204,6 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-queue", "Define a new stackRun by searching for warp IDs", - STACKTOOL_MODE_QUEUE, queueArgs); + PXOPT_ADD_MODE("-definebyquery", "Define a new stackRun by searching for warp IDs", + STACKTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); PXOPT_ADD_MODE("-definerun", "", STACKTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "", STACKTOOL_MODE_UPDATERUN, updaterunArgs);