IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Apr 19, 2013, 4:31:05 PM (13 years ago)
Author:
eugene
Message:

merge changes from eam_branches/ipp-20130307 : parallelize dvosecfilt, use dvo_average,measure,secfilt_init functions more broadly; move image calibration to relphot_images; add synthetic photometry referece (w-band); put relphot MARKTIME & INITTIME in macros; in dvo_clients, mextract fields which need image data use a subset image metadata table (instead of loading full Images.dat table); create FIELD and MOSAIC fields for mextract; somewhat better rules for photcode:ave & similar selections in mextract; add hpm_* concept in relastro (high-speed proper motions); fix precision errors in dvopsps; check for dvopsps exit conditions; error-bar clipping to limits of plotting window; parallelize delstar -dup-images; do NOT delete parents (because parent IDs are broken after dvomerge); minor handshake when setting up KAPA connection; avextract and related do not need to launch remote jobs on all clients if region does not include tables from all hosts; add fitplx and xsection functions to mana; enable addstar of diff cmfs (at least PS1_DV3)

Location:
trunk/Ohana
Files:
7 edited
3 copied

Legend:

Unmodified
Added
Removed

  • trunk/Ohana

  • trunk/Ohana/src/dvomerge/Makefile

    r34405 r35416  
    2323dvoverify_client : $(BIN)/dvoverify_client.$(ARCH)
    2424
    25 all: dvomerge dvomerge_client dvoconvert dvosecfilt dvoverify dvoverify_client
     25all: dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvoverify dvoverify_client
    2626
    2727#  $(SRC)/dvomergeContinue.$(ARCH).o
    … …  
    9393DVOSECFILT = \
    9494$(SRC)/dvosecfilt.$(ARCH).o \
     95$(SRC)/dvosecfilt_catalogs.$(ARCH).o \
    9596$(SRC)/SetSignals.$(ARCH).o \
    9697$(SRC)/ConfigInit.$(ARCH).o \
    … …  
    101102$(DVOSECFILT)  : $(INC)/dvomerge.h
    102103$(BIN)/dvosecfilt.$(ARCH) : $(DVOSECFILT)
     104
     105DVOSECFILT_CLIENT = \
     106$(SRC)/dvosecfilt_client.$(ARCH).o \
     107$(SRC)/dvosecfilt_catalogs.$(ARCH).o \
     108$(SRC)/SetSignals.$(ARCH).o \
     109$(SRC)/ConfigInit.$(ARCH).o \
     110$(SRC)/Shutdown.$(ARCH).o \
     111$(SRC)/help.$(ARCH).o \
     112$(SRC)/args.$(ARCH).o
     113
     114$(DVOSECFILT_CLIENT)  : $(INC)/dvomerge.h
     115$(BIN)/dvosecfilt_client.$(ARCH) : $(DVOSECFILT_CLIENT)
    103116
    104117DVOREPAIR = \
    … …  
    136149$(BIN)/dvoverify_client.$(ARCH) : $(DVOVERIFY_CLIENT)
    137150
    138 INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvorepair dvoverify dvoverify_client
     151INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client
    139152
    140153# dependancy rules for binary code #########################

  • trunk/Ohana/src/dvomerge/include/dvomerge.h

    r34405 r35416  
    3636int    REPLACE_BY_PHOTCODE;
    3737
     38char *SINGLE_CPT;
    3839SkyRegion UserPatch;  // used by MODE CAT
    3940
    … …  
    103104void       dvosecfilt_help        PROTO((int argc, char **argv));
    104105int        dvosecfilt_args        PROTO((int *argc, char **argv));
     106
     107void       dvosecfilt_client_usage PROTO((void));
     108void       dvosecfilt_client_help  PROTO((int argc, char **argv));
     109int        dvosecfilt_client_args  PROTO((int *argc, char **argv));
     110
     111int        dvosecfilt_catalogs     PROTO((int Nsecfilt));
     112int        dvosecfilt_parallel     PROTO((SkyTable *insky, int Nsecfilt));
    105113
    106114int        SkyTablePopulatedRange PROTO((off_t *ns, off_t *ne, SkyTable *sky, off_t Nstart));

  • trunk/Ohana/src/dvomerge/src/args.c

    r34260 r35416  
    170170  int N;
    171171
    172   /* extra error messages */
    173   VERBOSE = FALSE;
    174   if ((N = get_argument (*argc, argv, "-v"))) {
    175     VERBOSE = TRUE;
     172  HOST_ID = 0;
     173
     174  /* extra error messages */
     175  VERBOSE = FALSE;
     176  if ((N = get_argument (*argc, argv, "-v"))) {
     177    VERBOSE = TRUE;
     178    remove_argument (N, argc, argv);
     179  }
     180
     181  SINGLE_CPT = NULL;
     182  if ((N = get_argument (*argc, argv, "-cpt"))) {
     183    remove_argument (N, argc, argv);
     184    SINGLE_CPT = strcreate (argv[N]);
     185    remove_argument (N, argc, argv);
     186  }
     187
     188  /* specify portion of the sky : allow default of all sky? */
     189  UserPatch.Rmin = 0;
     190  UserPatch.Rmax = 360;
     191  UserPatch.Dmin = -90;
     192  UserPatch.Dmax = +90;
     193  if ((N = get_argument (*argc, argv, "-region"))) {
     194    remove_argument (N, argc, argv);
     195    UserPatch.Rmin = atof (argv[N]);
     196    remove_argument (N, argc, argv);
     197    UserPatch.Rmax = atof (argv[N]);
     198    remove_argument (N, argc, argv);
     199    UserPatch.Dmin = atof (argv[N]);
     200    remove_argument (N, argc, argv);
     201    UserPatch.Dmax = atof (argv[N]);
     202    remove_argument (N, argc, argv);
     203  }
     204
     205  PARALLEL = FALSE;
     206  if ((N = get_argument (*argc, argv, "-parallel"))) {
     207    PARALLEL = TRUE;
     208    remove_argument (N, argc, argv);
     209  }
     210  // this is a test mode : rather than launching the remote jobs and waiting for completion,
     211  // relphot will simply list the remote command and wait for the user to signal completion
     212  PARALLEL_MANUAL = FALSE;
     213  if ((N = get_argument (*argc, argv, "-parallel-manual"))) {
     214    PARALLEL = TRUE; // -parallel-manual implies -parallel
     215    PARALLEL_MANUAL = TRUE;
     216    remove_argument (N, argc, argv);
     217  }
     218  // this is a test mode : rather than launching the relphot_client jobs remotely, they are
     219  // run in serial via 'system'
     220  PARALLEL_SERIAL = FALSE;
     221  if ((N = get_argument (*argc, argv, "-parallel-serial"))) {
     222    if (PARALLEL_MANUAL) {
     223      fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n");
     224      exit (1);
     225    }
     226    PARALLEL = TRUE; // -parallel-serial implies -parallel
     227    PARALLEL_SERIAL = TRUE;
    176228    remove_argument (N, argc, argv);
    177229  }
    … …  
    180232  return TRUE;
    181233}
     234
     235/*** check for command line options ***/
     236int dvosecfilt_client_args (int *argc, char **argv) {
     237 
     238  int N;
     239
     240  // by definition, the client is not parallel
     241  PARALLEL = FALSE;
     242  PARALLEL_MANUAL = FALSE;
     243  PARALLEL_SERIAL = FALSE;
     244
     245  SINGLE_CPT = NULL;
     246  if ((N = get_argument (*argc, argv, "-cpt"))) {
     247    remove_argument (N, argc, argv);
     248    SINGLE_CPT = strcreate (argv[N]);
     249    remove_argument (N, argc, argv);
     250  }
     251
     252  /* specify portion of the sky : allow default of all sky? */
     253  UserPatch.Rmin = 0;
     254  UserPatch.Rmax = 360;
     255  UserPatch.Dmin = -90;
     256  UserPatch.Dmax = +90;
     257  if ((N = get_argument (*argc, argv, "-region"))) {
     258    remove_argument (N, argc, argv);
     259    UserPatch.Rmin = atof (argv[N]);
     260    remove_argument (N, argc, argv);
     261    UserPatch.Rmax = atof (argv[N]);
     262    remove_argument (N, argc, argv);
     263    UserPatch.Dmin = atof (argv[N]);
     264    remove_argument (N, argc, argv);
     265    UserPatch.Dmax = atof (argv[N]);
     266    remove_argument (N, argc, argv);
     267  }
     268
     269  HOST_ID = 0;
     270  if ((N = get_argument (*argc, argv, "-hostID"))) {
     271    remove_argument (N, argc, argv);
     272    HOST_ID = atoi (argv[N]);
     273    remove_argument (N, argc, argv);
     274  }
     275  if (!HOST_ID) dvosecfilt_client_usage();
     276
     277  HOSTDIR = NULL;
     278  if ((N = get_argument (*argc, argv, "-hostdir"))) {
     279    remove_argument (N, argc, argv);
     280    HOSTDIR = strcreate (argv[N]);
     281    remove_argument (N, argc, argv);
     282  }
     283  if (!HOSTDIR) dvosecfilt_client_usage();
     284
     285  /* extra error messages */
     286  VERBOSE = FALSE;
     287  if ((N = get_argument (*argc, argv, "-v"))) {
     288    VERBOSE = TRUE;
     289    remove_argument (N, argc, argv);
     290  }
     291
     292  if (*argc != 3) dvosecfilt_client_usage();
     293  return TRUE;
     294}

  • trunk/Ohana/src/dvomerge/src/dvosecfilt.c

    r29001 r35416  
    33// modify the number of average photcodes : change only secfilt tables in place
    44int main (int argc, char **argv) {
    5 
    6   char filename[256], *input;
    7   off_t i, j, k, Nsecfilt, NsecInput, Nstart;
    8   SkyTable *insky;
    9   Catalog catalog;
    10   SecFilt *insec, *outsec;
    115
    126  SetSignals ();
    … …  
    159  dvosecfilt_args (&argc, argv);
    1610
    17   input = argv[1];
    18   Nsecfilt = atoi(argv[2]);
     11  strcpy (CATDIR, argv[1]);
     12  int Nsecfilt = atoi(argv[2]);
    1913
    20   // load the sky table for the existing database
    21   insky = SkyTableLoadOptimal (input, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
    22   SkyTableSetFilenames (insky, input, "cpt");
    23 
    24   // loop over all input catalogs
    25   for (i = 0; i < insky[0].Nregions; i++) {
    26     if (!insky[0].regions[i].table) continue;
    27     if (VERBOSE) fprintf (stderr, "table: %s\n", insky[0].regions[i].name);
    28 
    29     // set the parameters which guide catalog open/load/create
    30     catalog.filename  = insky[0].filename[i];
    31     catalog.Nsecfilt  = 0;
    32 
    33     // always load all of the data (if any exists)
    34     catalog.catflags  = LOAD_SECF;
    35     catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
    36     catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
    37 
    38     if (!dvo_catalog_open (&catalog, &insky[0].regions[i], VERBOSE, "w")) {
    39         fprintf (stderr, "ERROR: failure to open catalog file %s\n", filename);
    40         exit (2);
    41     }
    42 
    43     if (catalog.Naves_disk == 0) {
    44       dvo_catalog_unlock (&catalog);
    45       dvo_catalog_free (&catalog);
    46       continue;
    47     }
    48 
    49     if (Nsecfilt == catalog.Nsecfilt) {
    50       dvo_catalog_unlock (&catalog);
    51       dvo_catalog_free (&catalog);
    52       continue;
    53     }
    54 
    55     NsecInput = catalog.Nsecfilt;
    56     Nstart = MIN(Nsecfilt, NsecInput);
    57 
    58     insec = catalog.secfilt;
    59     ALLOCATE (outsec, SecFilt, catalog.Naves_disk * Nsecfilt);
    60 
    61     for (k = 0; k < catalog.Naves_disk; k++) {
    62       for (j = 0; (j < catalog.Nsecfilt) && (j < Nsecfilt); j++) {
    63         outsec[k*Nsecfilt + j].M  = insec[k*NsecInput + j].M;
    64         outsec[k*Nsecfilt + j].dM = insec[k*NsecInput + j].dM;
    65         outsec[k*Nsecfilt + j].Xm = insec[k*NsecInput + j].Xm;
    66         outsec[k*Nsecfilt + j].flags  = insec[k*NsecInput + j].flags;
    67         outsec[k*Nsecfilt + j].Ncode  = insec[k*NsecInput + j].Ncode;
    68         outsec[k*Nsecfilt + j].Nused  = insec[k*NsecInput + j].Nused;
    69         outsec[k*Nsecfilt + j].M_20   = insec[k*NsecInput + j].M_20;
    70         outsec[k*Nsecfilt + j].M_80   = insec[k*NsecInput + j].M_80;
    71       }
    72       for (j = Nstart; j < Nsecfilt; j++) {
    73         outsec[k*Nsecfilt + j].M  = NAN;
    74         outsec[k*Nsecfilt + j].dM = NAN;
    75         outsec[k*Nsecfilt + j].Xm = NAN_S_SHORT;
    76       }
    77     }
    78     free (catalog.secfilt);
    79     catalog.secfilt = outsec;
    80     catalog.Nsecfilt = Nsecfilt;
    81     catalog.Nsecf_mem = Nsecfilt * catalog.Naves_disk;
    82     catalog.Nsecf_disk = Nsecfilt * catalog.Naves_disk;
    83 
    84     dvo_catalog_save (&catalog, VERBOSE);
    85     dvo_catalog_unlock (&catalog);
    86     dvo_catalog_free (&catalog);
    87   }
     14  dvosecfilt_catalogs (Nsecfilt);
    8815
    8916  exit (0);

  • trunk/Ohana/src/dvomerge/src/help.c

    r33963 r35416  
    2525void dvosecfilt_usage(void) {
    2626
    27   fprintf (stderr, "USAGE: dvosecfilt (catdir) (Nsecfilt)\n");
     27  fprintf (stderr, "USAGE: dvosecfilt (catdir) (Nsecfilt)\n\n");
     28  fprintf (stderr, "  optional flags:\n");
     29  fprintf (stderr, "  -v                          : verbose mode\n");
     30
     31  exit (2);
     32}
     33
     34void dvosecfilt_client_usage(void) {
     35
     36  fprintf (stderr, "USAGE: dvosecfilt_client (catdir) (Nsecfilt)\n\n");
     37  fprintf (stderr, "  optional flags:\n");
     38  fprintf (stderr, "  -v                          : verbose mode\n");
    2839
    2940  exit (2);
    … …  
    123134}
    124135
     136void dvosecfilt_client_help (int argc, char **argv) {
     137
     138  /* check for help request */
     139  if (get_argument (argc, argv, "-help")) goto show_help;
     140  if (get_argument (argc, argv, "-h"))    goto show_help;
     141  return;
     142
     143show_help:
     144
     145  fprintf (stderr, "USAGE\n");
     146  fprintf (stderr, "  dvosecfilt_client (catdir) (Nsecfilt)\n\n");
     147
     148  fprintf (stderr, "  change number of secfilt entries in photcode table (updates secfilt tables (cps), NSECFILT in cpt files)\n");
     149  fprintf (stderr, "  NOTE: the user must change the photcode table to reflect the change (use photcode-table -export / -import)\n");
     150 
     151  fprintf (stderr, "  optional flags:\n");
     152  fprintf (stderr, "  -v                          : verbose mode\n");
     153  fprintf (stderr, "  -help                       : this list\n");
     154  fprintf (stderr, "  -h                          : this list\n\n");
     155  exit (2);
     156}
     157
    125158void dvorepair_help (int argc, char **argv) {
    126159

  • trunk/Ohana/src/dvomerge/src/merge_catalogs_old.c

    r34405 r35416  
    299299
    300300    // XXX should we accept the input measurements for these fields?
    301 
     301    dvo_average_init (&output[0].average[Nave]);
    302302    output[0].average[Nave].R              = input[0].average[N].R;
    303303    output[0].average[Nave].D              = input[0].average[N].D;
    304     output[0].average[Nave].dR             = 0;
    305     output[0].average[Nave].dD             = 0;
    306 
    307     output[0].average[Nave].Nmeasure       = 0; // this value is update as the measurements are associated with this entry below
    308     output[0].average[Nave].Nmissing       = 0;
    309     output[0].average[Nave].Nextend        = 0;
    310 
    311304    output[0].average[Nave].measureOffset  = Nmeas;
    312     output[0].average[Nave].missingOffset  = -1;
    313     output[0].average[Nave].extendOffset   = -1;
    314 
    315     output[0].average[Nave].uR             = 0;
    316     output[0].average[Nave].uD             = 0;
    317     output[0].average[Nave].duR            = 0;
    318     output[0].average[Nave].duD            = 0;
    319     output[0].average[Nave].P              = 0;
    320     output[0].average[Nave].dP             = 0;
    321 
    322     output[0].average[Nave].stargal        = 0;
    323     output[0].average[Nave].ChiSqAve       = 0.0;
    324     output[0].average[Nave].ChiSqPM        = 0.0;
    325     output[0].average[Nave].ChiSqPar       = 0.0;
    326     output[0].average[Nave].Tmean          = 0;
    327     output[0].average[Nave].Trange         = 0;
    328     output[0].average[Nave].Npos           = 0;
    329 
    330305    output[0].average[Nave].objID          = objID; // we create objID values in the context of the output db
    331306    output[0].average[Nave].catID          = catID; // we create catID values in the context of the output db
    332     output[0].average[Nave].flags          = 0;  // XXX why reset these?
     307
    333308    if (PSPS_ID) {
    334309      output[0].average[Nave].extID = CreatePSPSObjectID(output[0].average[Nave].R, output[0].average[Nave].D);
    335     } else {
    336       output[0].average[Nave].extID = 0;
    337     }
     310    }
    338311
    339312    objID ++;
    340313
     314    // init the new secfilt entries
    341315    for (j = 0; j < NsecfiltOut; j++) {
    342316        int outputIndex = (Nave * NsecfiltOut) + j;
    343         output[0].secfilt[outputIndex].M     = NAN;
    344         output[0].secfilt[outputIndex].dM    = NAN;
    345         output[0].secfilt[outputIndex].Xm    = NAN_S_SHORT;
    346         output[0].secfilt[outputIndex].M_20  = NAN_S_SHORT;
    347         output[0].secfilt[outputIndex].M_80  = NAN_S_SHORT;
    348         output[0].secfilt[outputIndex].Ncode = 0;
    349         output[0].secfilt[outputIndex].Nused = 0;
     317        dvo_secfilt_init (&output[0].secfilt[outputIndex]);
    350318    }
    351319
    … …  
    359327   
    360328      if (isfinite(input[0].secfilt[N*NsecfiltIn+j].M)) {
    361         output[0].secfilt[outputIndex].M     = input[0].secfilt[N*NsecfiltIn+j].M;
    362         output[0].secfilt[outputIndex].dM    = input[0].secfilt[N*NsecfiltIn+j].dM;
    363         output[0].secfilt[outputIndex].Xm    = input[0].secfilt[N*NsecfiltIn+j].Xm;
    364         output[0].secfilt[outputIndex].M_20  = input[0].secfilt[N*NsecfiltIn+j].M_20;
    365         output[0].secfilt[outputIndex].M_80  = input[0].secfilt[N*NsecfiltIn+j].M_80;
    366         output[0].secfilt[outputIndex].Ncode = input[0].secfilt[N*NsecfiltIn+j].Ncode;
    367         output[0].secfilt[outputIndex].Nused = input[0].secfilt[N*NsecfiltIn+j].Nused;
     329        output[0].secfilt[outputIndex] = input[0].secfilt[N*NsecfiltIn+j];
    368330      }
    369331    }
Note: See TracChangeset for help on using the changeset viewer.