IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 37963


Ignore:
Timestamp:
Mar 11, 2015, 6:34:45 AM (11 years ago)
Author:
eugene
Message:

more logging info for remote jobs

Location:
branches/eam_branches/ipp-20150112/Ohana/src/relastro
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20150112/Ohana/src/relastro/include/relastro.h

    r37807 r37963  
    99// # define IDX_T off_t
    1010# define IDX_T int
     11
     12# define LOGRTIME(MSG,...) {                            \
     13    gettimeofday (&stopTimer, (void *) NULL);           \
     14    float dtime = DTIME (stopTimer, startTimer);        \
     15    client_logger_message (MSG, __VA_ARGS__); }
    1116
    1217typedef enum {

  • branches/eam_branches/ipp-20150112/Ohana/src/relastro/src/MeasPosIO.c

    r37807 r37963  
    1919  MeasPos *measpos = NULL;
    2020
     21  INITTIME;
     22
    2123  FILE *f = fopen (filename, "r");
    2224  if (!f) {
    … …  
    4648  }
    4749
     50  LOGRTIME("MeasPosLoad_init %s: %f sec\n", filename, dtime);
     51
    4852  // read the fits table bytes
    4953  if (!gfits_fread_ftable_data (f, &ftable, FALSE)) {
    … …  
    5256  }
    5357  fclose (f);
     58  LOGRTIME("MeasPosLoad_read %s: %f sec\n", filename, dtime);
    5459
    5560  // a bit annoying : we read the entire block of data, then extract the columns, then set the image structure values.
    … …  
    6368  GET_COLUMN (catID,     "CAT_ID",       int);
    6469  GET_COLUMN (imageID,   "IMAGE_ID",     int);
     70  LOGRTIME("MeasPosLoad_getcol %s: %f sec\n", filename, dtime);
    6571
    6672  ALLOCATE (measpos, MeasPos, Nrow);
     73  LOGRTIME("MeasPosLoad_alloc %s: %f sec\n", filename, dtime);
     74
    6775  for (i = 0; i < Nrow; i++) {
    6876    measpos[i].R              = R    [i];
    … …  
    7280    measpos[i].imageID        = imageID[i];
    7381  }
     82  LOGRTIME("MeasPosLoad_convert %s: %f sec\n", filename, dtime);
     83
    7484  fprintf (stderr, "loaded data for %lld objects (* filters)\n", (long long) Nrow);
    7585
    … …  
    8595  gfits_free_header (&theader);
    8696  gfits_free_table  (&ftable);
     97  LOGRTIME("MeasPosLoad_cleanup %s: %f sec\n", filename, dtime);
    8798
    8899  *nmeaspos = Nrow;

  • branches/eam_branches/ipp-20150112/Ohana/src/relastro/src/relastro_parallel_images.c

    r37807 r37963  
    2222  RegionHostTable *regionHosts = RegionHostTableLoad (CATDIR, REGION_FILE);
    2323  int myHost = regionHosts->index[REGION_HOST_ID];
     24  char *myHostName = regionHosts->hosts[myHost].hostname;
     25
    2426  RegionHostFindNeighbors (regionHosts, myHost);
    25   client_logger_message ("started parallel images on %s\n", regionHosts->hosts[myHost].hostname);
     27  client_logger_message ("started parallel images on %s\n", myHostName);
    2628
    2729  // load the subset images belonging to this host
    … …  
    4749  }
    4850  put_astrom_table (table);
    49   client_logger_message ("loaded images on %s\n", regionHosts->hosts[myHost].hostname);
     51  client_logger_message ("loaded images on %s\n", myHostName);
    5052
    5153  // once we have read this table, we should remove it for repeat runs
    … …  
    6264
    6365  /* load catalog data from region files (hostID is 0 since we are not a client */
    64   char *syncfile = make_filename (CATDIR, regionHosts->hosts[myHost].hostname, REGION_HOST_ID, "loadcat.sync");
     66  char *syncfile = make_filename (CATDIR, myHostName, REGION_HOST_ID, "loadcat.sync");
    6567  catalog = load_catalogs (skylist, &Ncatalog, TRUE, 0, NULL, syncfile);
    6668  MARKTIME("-- load catalog data, host %d: %f sec\n", REGION_HOST_ID, dtime);
    6769  free (syncfile);
    6870 
    69   client_logger_message ("loaded catalog data %s\n", regionHosts->hosts[myHost].hostname);
     71  client_logger_message ("loaded catalog data %s\n", myHostName);
    7072
    7173  // find ICRF QSOs for reference downstream (only if USE_ICRF_CORRECT)
    … …  
    9092  createStarMap (catalog, Ncatalog);
    9193
    92   client_logger_message ("set up image indexes on %s\n", regionHosts->hosts[myHost].hostname);
     94  client_logger_message ("set up image indexes on %s\n", myHostName);
    9395
    9496  markObjects (catalog, Ncatalog);
    … …  
    9698  SAVEPLOT = FALSE;
    9799
    98   client_logger_message ("starting the loops: %s\n", regionHosts->hosts[myHost].hostname);
     100  client_logger_message ("starting the loops: %s\n", myHostName);
    99101
    100102  /* major modes */
    … …  
    114116      if (RESET_IMAGES) {
    115117        UpdateMeasures (catalog, Ncatalog);
    116         MARKTIME("reset measures, host %d: %f sec\n", REGION_HOST_ID, dtime);
    117         client_logger_message ("reset measures: %s\n", regionHosts->hosts[myHost].hostname);
     118        MARKTIME("UpdateMeasures on %s, host %d: %f sec\n", myHostName, REGION_HOST_ID, dtime);
     119        LOGRTIME("UpdateMeasures on %s, host %d: %f sec\n", myHostName, REGION_HOST_ID, dtime);
    118120      }
    119121      for (i = 0; i < NLOOP; i++) {
    120122        UpdateObjects (catalog, Ncatalog, (i > 0));
    121         MARKTIME("update objects loop %d, host %d: %f sec\n", i, REGION_HOST_ID, dtime);
    122         client_logger_message ("update objects loop %d: %s\n", i, regionHosts->hosts[myHost].hostname);
     123        LOGRTIME("UpdateObjects loop %d on %s, host %d: %f sec\n", i, myHostName, REGION_HOST_ID, dtime);
    123124        if ((i > 1) || !USE_GALAXY_MODEL) {
    124125          // if GALAXY_MODEL is selected, we want to delay the frame correction until we have
    125126          // applied the galaxy model a couple of times.
    126127          FrameCorrectionParallelSlave (catalog, Ncatalog, regionHosts, i);
    127           MARKTIME("frame correction loop %d, host %d: %f sec\n", i, REGION_HOST_ID, dtime);
    128           client_logger_message ("frame correction loop %d: %s\n", i, regionHosts->hosts[myHost].hostname);
     128          LOGRTIME("FrameCorrection loop %d on %s, host %d: %f sec\n", i, myHostName, REGION_HOST_ID, dtime);
    129129        }
    130130        share_mean_pos (catalog, Ncatalog, regionHosts, i);
     131        LOGRTIME("share_mean_pos loop %d on %s, host %d: %f sec\n", i, myHostName, REGION_HOST_ID, dtime);
    131132        slurp_mean_pos (catalog, Ncatalog, regionHosts, i);
    132         client_logger_message ("slurp mags loop %d: %s\n", i, regionHosts->hosts[myHost].hostname);
     133        LOGRTIME("slurp_mean_pos loop %d on %s, host %d: %f sec\n", i, myHostName, REGION_HOST_ID, dtime);
    133134
    134135        UpdateChips (catalog, Ncatalog);
    135         MARKTIME("update chips loop %d, host %d: %f sec\n", i, REGION_HOST_ID, dtime);
    136         client_logger_message ("update chips loop %d: %s\n", i, regionHosts->hosts[myHost].hostname);
     136        LOGRTIME("UpdateChips loop %d on %s, host %d: %f sec\n", i, myHostName, REGION_HOST_ID, dtime);
    137137
    138138        share_image_pos (regionHosts, i);
     139        LOGRTIME("share_image_pos loop on %d on %s, host %d: %f sec\n", i, myHostName, REGION_HOST_ID, dtime);
    139140        slurp_image_pos (catalog, Ncatalog, regionHosts, i);
    140         client_logger_message ("slurp image pos loop %d: %s\n", i, regionHosts->hosts[myHost].hostname);
     141        LOGRTIME("slurp_image_pos loop on %d on %s, host %d: %f sec\n", i, myHostName, REGION_HOST_ID, dtime);
    141142
    142143        share_meas_pos (catalog, Ncatalog, regionHosts, i);
     144        LOGRTIME("share_meas_pos loop %d on %s, host %d: %f sec\n", i, myHostName, REGION_HOST_ID, dtime);
    143145        slurp_meas_pos (catalog, Ncatalog, regionHosts, i);
    144         MARKTIME("done exchange loop %d, host %d: %f sec\n", i, REGION_HOST_ID, dtime);
    145         client_logger_message ("slurp meas loop %d: %s\n", i, regionHosts->hosts[myHost].hostname);
     146        LOGRTIME("slurp_meas_pos loop %d on %s, host %d: %f sec\n", i, myHostName, REGION_HOST_ID, dtime);
    146147      }
    147148      // create summary plots of the process
    … …  
    166167
    167168  // this is a checkpoint to make sure all hosts have finished the loop above
    168   char *loopsyncfile = make_filename (CATDIR, regionHosts->hosts[myHost].hostname, REGION_HOST_ID, "loop.sync");
     169  char *loopsyncfile = make_filename (CATDIR, myHostName, REGION_HOST_ID, "loop.sync");
    169170  update_sync_file (loopsyncfile, 0);
    170171  free (loopsyncfile);
    … …  
    178179
    179180  share_image_pos (regionHosts, -1);
    180   client_logger_message ("share image pos loop %d: %s\n", -1, regionHosts->hosts[myHost].hostname);
     181  LOGRTIME("share image pos loop %d on %s, host %d: %f sec\n", -1, myHostName, REGION_HOST_ID, dtime);
    181182
    182183  exit (0);

  • branches/eam_branches/ipp-20150112/Ohana/src/relastro/src/share_images_pos.c

    r37807 r37963  
    6060  ALLOCATE (image_pos, ImagePos, 1);
    6161
     62  INITTIME;
     63  int myHost = regionHosts->index[REGION_HOST_ID];
     64  char *myHostName = regionHosts->hosts[myHost].hostname;
     65
    6266  fprintf (stderr, "grabbing image mags from other hosts...\n");
    6367
     68  LOGRTIME("image_load_start loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    6469  for (i = 0; i < regionHosts->Nhosts; i++) {
    6570    if (regionHosts->hosts[i].hostID == REGION_HOST_ID) continue;
    … …  
    6974    check_sync_file (syncfile, nloop);
    7075    free (syncfile);
     76    LOGRTIME("image_load_sync host %d loop %d on %s, host %d: %f sec\n", i, nloop, myHostName, REGION_HOST_ID, dtime);
    7177   
    7278    off_t Nsubset;
    … …  
    7480    ImagePos *image_pos_subset = ImagePosLoad (iposfile, &Nsubset);
    7581    free (iposfile);
     82    LOGRTIME("image_load_read host %d loop %d on %s, host %d: %f sec\n", i, nloop, myHostName, REGION_HOST_ID, dtime);
    7683
    7784    image_pos = merge_image_pos (image_pos, &Nimage_pos, image_pos_subset, Nsubset);
     85    LOGRTIME("image_load_merge host %d loop %d on %s, host %d: %f sec\n", i, nloop, myHostName, REGION_HOST_ID, dtime);
    7886  }
    7987
    … …  
    96104    images[seq].flags        = image_pos[i].flags     ;
    97105  }
     106  LOGRTIME("image_load_convert loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    98107
    99108  // load the astrometry offset maps, if they exist, and apply
    … …  
    106115   
    107116    AstromOffsetTable *table = AstromOffsetMapLoad (mapname, VERBOSE);
     117    LOGRTIME("image_maps_load host %d loop %d on %s, host %d: %f sec\n", i, nloop, myHostName, REGION_HOST_ID, dtime);
    108118 
    109119    // apply table entries here to existing images
    … …  
    127137      }
    128138    }
     139    LOGRTIME("image_maps_copy host %d loop %d on %s, host %d: %f sec\n", i, nloop, myHostName, REGION_HOST_ID, dtime);
     140
    129141    AstromOffsetTableFree(table);
     142    LOGRTIME("image_maps_free host %d loop %d on %s, host %d: %f sec\n", i, nloop, myHostName, REGION_HOST_ID, dtime);
    130143  }
    131144
    … …  
    137150  }
    138151  free (image_pos);
     152  LOGRTIME("image_load_apply loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    139153
    140154  fprintf (stderr, "DONE grabbing image mags from other hosts\n");

  • branches/eam_branches/ipp-20150112/Ohana/src/relastro/src/share_mean_pos.c

    r37038 r37963  
    1515
    1616  int myHost = regionHosts->index[REGION_HOST_ID];
     17  char *myHostName = regionHosts->hosts[myHost].hostname;
     18
    1719  double Rmin = regionHosts->hosts[myHost].Rmin;
    1820  double Rmax = regionHosts->hosts[myHost].Rmax;
    1921  double Dmin = regionHosts->hosts[myHost].Dmin;
    2022  double Dmax = regionHosts->hosts[myHost].Dmax;
     23
     24  INITTIME;
    2125
    2226  // XXX skip some catalogs based on UserPatch?
    … …  
    4044    }
    4145  }
     46  LOGRTIME("set_mean_pos loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    4247
    4348  // write out the meanmag fits table AND write state in some file
    … …  
    4853  free (meanpos);
    4954  free (posfile);
     55  LOGRTIME("MeanPosSave loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    5056
    5157  char *syncfile = make_filename (CATDIR, hostname, REGION_HOST_ID, "meanpos.sync");
    5258  update_sync_file (syncfile, nloop);
    5359  free (syncfile);
     60  LOGRTIME("update_sync_file loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    5461
    5562  return TRUE;
    … …  
    6471  ALLOCATE (meanpos, MeanPos, 1);
    6572
     73  int myHost = regionHosts->index[REGION_HOST_ID];
     74  char *myHostName = regionHosts->hosts[myHost].hostname;
     75
    6676  fprintf (stderr, "grabbing mean object pos from other hosts...\n");
     77
     78  INITTIME;
    6779
    6880  for (i = 0; i < regionHosts->Nhosts; i++) {
    … …  
    8294    meanpos = merge_mean_pos (meanpos, &Nmeanpos, meanposSubset, Nsubset);
    8395  }
     96  LOGRTIME("MeanPosLoad/merge_mean_pos loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    8497
    8598  for (i = 0; i < Nmeanpos; i++) {
    … …  
    99112  }
    100113  free (meanpos);
     114  LOGRTIME("match_catID_and_objID loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    101115
    102116  fprintf (stderr, "DONE grabbing mean object pos from other hosts...\n");

  • branches/eam_branches/ipp-20150112/Ohana/src/relastro/src/share_meas_pos.c

    r37807 r37963  
    1414  ALLOCATE (measpos, MeasPos, NMEASPOS);
    1515
     16  INITTIME;
    1617  int myHost = regionHosts->index[REGION_HOST_ID];
     18  char *myHostName = regionHosts->hosts[myHost].hostname;
    1719
    1820  // XXX skip some catalogs based on UserPatch?
    … …  
    3335    }
    3436  }
     37  LOGRTIME("set_meas_pos loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    3538
    3639  // write out the measmag fits table AND write state in some file
    … …  
    4144  free (measpos);
    4245  free (posfile);
     46  LOGRTIME("MeasPosSave loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    4347
    4448  char *syncfile = make_filename (CATDIR, hostname, REGION_HOST_ID, "measpos.sync");
    4549  update_sync_file (syncfile, nloop);
    4650  free (syncfile);
     51  LOGRTIME("update_sync_file loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    4752
    4853  return TRUE;
    … …  
    5964  fprintf (stderr, "grabbing meas object pos from other hosts...\n");
    6065
     66  INITTIME;
     67  int myHost = regionHosts->index[REGION_HOST_ID];
     68  char *myHostName = regionHosts->hosts[myHost].hostname;
     69
     70  LOGRTIME("meas_load_start loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    6171  for (i = 0; i < regionHosts->Nhosts; i++) {
    6272    if (regionHosts->hosts[i].hostID == REGION_HOST_ID) continue;
    … …  
    6676    check_sync_file (syncfile, nloop);
    6777    free (syncfile);
     78    LOGRTIME("meas_load_sync host %d loop %d on %s, host %d: %f sec\n", i, nloop, myHostName, REGION_HOST_ID, dtime);
    6879   
    6980    off_t Nsubset = 0;
    … …  
    7182    MeasPos *measposSubset = MeasPosLoad (posfile, &Nsubset);
    7283    free (posfile);
     84    LOGRTIME("MeasPosLoad host %d loop %d on %s, host %d: %f sec\n", i, nloop, myHostName, REGION_HOST_ID, dtime);
    7385
    7486    // merge_meas_pos reallocs measpos and frees the input measposSubset
    7587    measpos = merge_meas_pos (measpos, &Nmeaspos, measposSubset, Nsubset);
     88    LOGRTIME("merge_meas_pos host %d loop %d on %s, host %d: %f sec\n", i, nloop, myHostName, REGION_HOST_ID, dtime);
    7689  }
     90  LOGRTIME("meas_load_done loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    7791
    7892  for (i = 0; i < Nmeaspos; i++) {
    … …  
    99113  }
    100114  free (measpos);
     115  LOGRTIME("match_catID_and_objID/meas loop %d on %s, host %d: %f sec\n", nloop, myHostName, REGION_HOST_ID, dtime);
    101116
    102117  fprintf (stderr, "DONE grabbing meas object pos from other hosts...\n");
Note: See TracChangeset for help on using the changeset viewer.