IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 40114


Ignore:
Timestamp:
Aug 9, 2017, 12:51:03 PM (9 years ago)
Author:
eugene
Message:

major mods to track down dvopsps segfaults

Location:
branches/eam_branches/ohana.20170809/src/dvopsps
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ohana.20170809/src/dvopsps/include/dvopsps.h

    r39112 r40114  
    146146Detections *DetectionsLoad                PROTO((char *filename, int *Ndetections));
    147147int         DetectionsSave                PROTO((char *filename, Detections *detections, int Ndetections));
     148void dvopsps_client_free (void);

  • branches/eam_branches/ohana.20170809/src/dvopsps/src/DetectionOps.c

    r39547 r40114  
    151151  FTable ftable;
    152152
    153   gfits_init_header (&header);
     153  gfits_init_header (&header); // *** FUNC : OK
    154154  header.extend = TRUE;
    155   gfits_create_header (&header);
    156   gfits_create_matrix (&header, &matrix);
     155  gfits_create_header (&header); // *** FUNC : OK
     156  gfits_create_matrix (&header, &matrix); // *** FUNC : OK
    157157
    158158  FILE *f = fopen (filename, "w");
    … …  
    162162  }
    163163
    164   gfits_fwrite_header  (f, &header);
    165   gfits_fwrite_matrix  (f, &matrix);
    166   gfits_free_header (&header);
    167   gfits_free_matrix (&matrix);
    168 
    169   gfits_create_table_header (&theader, "BINTABLE", "DETECTIONS");
     164  gfits_fwrite_header  (f, &header); // *** FUNC : OK
     165  gfits_fwrite_matrix  (f, &matrix); // *** FUNC : OK
     166  gfits_free_header (&header); // *** FUNC : OK
     167  gfits_free_matrix (&matrix); // *** FUNC : OK
     168
     169  gfits_create_table_header (&theader, "BINTABLE", "DETECTIONS"); // *** FUNC : OK
    170170
    171171  // XXX need to get the bzero values right
    172   gfits_define_bintable_column (&theader, "K", "objID",       NULL, NULL, 1.0, 0);
     172  gfits_define_bintable_column (&theader, "K", "objID",       NULL, NULL, 1.0, 0); // *** FUNC : OK (also below)
    173173  gfits_define_bintable_column (&theader, "K", "detectID",    NULL, NULL, 1.0, 0);
    174174  gfits_define_bintable_column (&theader, "K", "ippObjID",    NULL, NULL, 1.0, 0);
    … …  
    195195
    196196  // generate the output array that carries the data
    197   gfits_create_table (&theader, &ftable);
     197  gfits_create_table (&theader, &ftable); // *** FUNC : OK
     198  gfits_set_table_rows (&theader, &ftable, Ndetections); // *** FUNC : OK
    198199
    199200  // create intermediate storage arrays
    … …  
    302303  gfits_free_header (&theader);
    303304  gfits_free_table (&ftable);
     305
     306  fclose (f);
    304307  return TRUE;
    305308}

  • branches/eam_branches/ohana.20170809/src/dvopsps/src/dvopsps_client.c

    r38674 r40114  
    1414
    1515  // get configuration info, args, lockfile (set CATDIR, HOST_ID, HOSTDIR, IMAGES)
    16   initialize_dvopsps_client (argc, argv);
     16  // initialize_dvopsps_client (argc, argv);
     17
     18  TIME_START = NULL;
     19  TIME_END = NULL;
    1720
    1821  if (!strcasecmp (argv[1], "detections")) {
    … …  
    3639  }
    3740
     41  dvopsps_client_free ();
     42
     43  ohana_memcheck (TRUE);
     44  ohana_memdump (TRUE);
    3845  if (!status) exit (1);
    3946  exit (0);

  • branches/eam_branches/ohana.20170809/src/dvopsps/src/initialize_dvopsps.c

    r38674 r40114  
    222222void initialize_dvopsps_client (int argc, char **argv) {
    223223
     224  TIME_START = NULL;
     225  TIME_END = NULL;
     226  return;
     227
    224228  int N;
    225229  struct stat statbuffer;
    … …  
    231235  if ((N = get_argument (argc, argv, "--help"))) usage_dvopsps_client();
    232236
    233   SetZeroPoint (25.0); // XXX is this needed?
    234   args_dvopsps_client (argc, argv);
     237  TIME_START = NULL;
     238  TIME_END = NULL;
     239
     240  // SetZeroPoint (25.0); // XXX is this needed?
     241  // args_dvopsps_client (argc, argv);
     242  return;
    235243
    236244  if (stat (CATDIR, &statbuffer)) {
    … …  
    256264  PARALLEL_SERIAL = FALSE;
    257265
     266  DATABASE_HOST = NULL;
    258267  if ((N = get_argument (argc, argv, "-dbhost"))) {
    259268    remove_argument (N, &argc, argv);
    260269    DATABASE_HOST = strcreate (argv[N]);
    261270    remove_argument (N, &argc, argv);
    262   } else usage_dvopsps_client();
    263 
     271  } // else usage_dvopsps_client();
     272
     273  DATABASE_USER = NULL;
    264274  if ((N = get_argument (argc, argv, "-dbuser"))) {
    265275    remove_argument (N, &argc, argv);
    266276    DATABASE_USER = strcreate (argv[N]);
    267277    remove_argument (N, &argc, argv);
    268   } else usage_dvopsps_client();
    269 
     278  } // else usage_dvopsps_client();
     279
     280  DATABASE_PASS = NULL;
    270281  if ((N = get_argument (argc, argv, "-dbpass"))) {
    271282    remove_argument (N, &argc, argv);
    272283    DATABASE_PASS = strcreate (argv[N]);
    273284    remove_argument (N, &argc, argv);
    274   } else usage_dvopsps_client();
    275 
     285  } // else usage_dvopsps_client();
     286
     287  DATABASE_NAME = NULL;
    276288  if ((N = get_argument (argc, argv, "-dbname"))) {
    277289    remove_argument (N, &argc, argv);
    278290    DATABASE_NAME = strcreate (argv[N]);
    279291    remove_argument (N, &argc, argv);
    280   } else usage_dvopsps_client();
     292  } // else usage_dvopsps_client();
    281293
    282294  RESULT_FILE = NULL; // only used by dvopsps_client
    … …  
    299311  UserPatch.Dmin = -90;
    300312  UserPatch.Dmax = +90;
    301   if ((N = get_argument (argc, argv, "-region"))) {
     313
     314  UserPatch.Rmin = 277.594177;
     315  UserPatch.Rmax = 279.405823;
     316  UserPatch.Dmin =  -7.400000;
     317  UserPatch.Dmax =  -5.600000;
     318
     319  if (FALSE && (N = get_argument (argc, argv, "-region"))) {
    302320    remove_argument (N, &argc, argv);
    303321    UserPatch.Rmin = atof (argv[N]);
    … …  
    361379  }
    362380
    363   HOST_ID = 0;
     381  HOST_ID = 1; // XXXX was 0
    364382  if ((N = get_argument (argc, argv, "-hostID"))) {
    365383    remove_argument (N, &argc, argv);
    … …  
    375393    remove_argument (N, &argc, argv);
    376394  }
    377   if (!HOSTDIR) usage_dvopsps_client();
     395  // XXXX TEMP REMOVE if (!HOSTDIR) usage_dvopsps_client();
    378396
    379397  if ((N = get_argument (argc, argv, "-catdir"))) {
    … …  
    382400    remove_argument (N, &argc, argv);
    383401  }
    384   if (!CATDIR) usage_dvopsps_client();
    385 
    386   if (argc != 2) usage_dvopsps_client();
     402  // if (!CATDIR) usage_dvopsps_client();
     403
     404  // if (argc != 2) usage_dvopsps_client();
    387405  return (TRUE);
     406}
     407
     408void dvopsps_client_free (void) {
     409
     410  FREE (DATABASE_HOST);
     411  FREE (DATABASE_USER);
     412  FREE (DATABASE_PASS);
     413  FREE (DATABASE_NAME);
     414 
     415  FREE (RESULT_FILE);
     416  FREE (SINGLE_CPT);
     417  FREE (TIME_START);
     418  FREE (TIME_END);
     419 
     420  FREE (HOSTDIR);
     421  FREE (CATDIR);
    388422}
    389423

  • branches/eam_branches/ohana.20170809/src/dvopsps/src/insert_detections_dvopsps.c

    r38471 r40114  
    11# include "dvopsps.h"
    2 # define USE_MYSQL 1
    3 // # define SAVE_REMOTE 1
    4 
    5 // we have two ways of writing to the mysql database:
    6 // SAVE_REMOTE = 0 : each client sends the detections directly to the mysql server
    7 // SAVE_REMOTE = 1 : each client saves the detections to disk and these are loaded by the main program and sent to the mysql server
    82
    93// determine the relevant catalogs, launch parallel clients if desired
    104int insert_detections_dvopsps () {
    115
    12   SkyTable *sky = NULL;
    13   SkyList *skylist = NULL;
    146  Catalog catalog;
    15   off_t i;
    16 
    17   // load the current sky table (layout of all SkyRegions)
    18   sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, -1, VERBOSE);
    19   SkyTableSetFilenames (sky, CATDIR, "cpt");
    20  
    21   if (PARALLEL && !HOST_ID) {
    22     int status = insert_detections_dvopsps_parallel (sky);
    23     return status;
    24   }
    25 
    26   // determine the populated SkyRegions overlapping the requested area (default depth)
    27   if (SINGLE_CPT) {
    28       skylist = SkyRegionByCPT (sky, SINGLE_CPT);
    29   } else {
    30       skylist = SkyListByPatch (sky, -1, &UserPatch);
    31   }
    32   myAssert (skylist, "ooops!");
    33 
    34 # if (USE_MYSQL)
    35   // NOTE: mysql connection happens here since each dvopsps_client makes its own connection
    36   MYSQL  mysqlBase;
    37   MYSQL *mysqlReal = NULL;
    38   if (!HOST_ID || !SAVE_REMOTE) {
    39     mysqlReal = mysql_dvopsps_connect (&mysqlBase);
    40     if (!mysqlReal) {
    41       fprintf (stderr, "failed to connect to mysql\n");
    42       exit (1);
    43     }
    44   }
    45 # else
    46   MYSQL *mysqlReal = NULL;
    47 # endif
    48 
    49   if (SAVE_REMOTE && HOST_ID) {
    50     init_detections ();
    51   }
     7
     8  init_detections (); // *** FUNC : OK
    529
    5310  int status = TRUE;
    54   // select measurements for each populated catalog
    55   for (i = 0; i < skylist[0].Nregions; i++) {
    56 
    57     // does this host ID match the desired location for the table?
    58     if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue;
    59 
    60     char localFilename[1024];
    61     snprintf (localFilename, 1024, "%s/%s.cpt", HOSTDIR, skylist->regions[i]->name);
    62 
    63     // set up the basic catalog info
    64     dvo_catalog_init (&catalog, TRUE);
    65     catalog.filename  = HOST_ID ? localFilename : skylist[0].filename[i];
    66     catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT;
    67     catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
    68 
    69     if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "r")) {
    70       fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename);
    71       exit (1);
    72     }
    73     if (!catalog.Naverage_disk) {
    74       if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename);
    75       dvo_catalog_unlock (&catalog);
    76       dvo_catalog_free (&catalog);
    77       continue;
    78     }
    79 
    80     // NOTE: this is where the real action happens
    81     if (SAVE_REMOTE && HOST_ID) {
    82       if (!append_detections_dvopsps_catalog (&catalog)) {
    83         fprintf (stderr, "ERROR: failure to append detections to output catalog\n");
    84         status = FALSE;
    85       }
    86     } else {
    87       if (!insert_detections_dvopsps_catalog (&catalog, mysqlReal)) {
    88         fprintf (stderr, "ERROR: failure to insert detections into mysql database\n");
    89         status = FALSE;
    90       }
    91     }
    92 
    93     // NOTE : unlike setastrom or relphot, this program is read-only wrt dvo
     11
     12  char *localFilename = NULL; // XXX stops aborting if I drop this (probably affects
     13                              // memory position) (FREE below @ line 40 is also required)
     14
     15  // set up the basic catalog info
     16  dvo_catalog_init (&catalog, TRUE);  // *** FUNC : OK
     17//catalog.filename  = strcreate ("/data/ipp110.0/eugene/3pi.dvo.catdirs/3pi.pv3.20170216/catdir.037/s0000/5124.14.cpt"); // *** FUNC : OK
     18//catalog.filename  = strcreate ("z0/5124.14.cpt"); // *** FUNC : OK
     19  catalog.filename  = strcreate ("v0/5124.14.cpt"); // *** FUNC : OK
     20  catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT;
     21  catalog.Nsecfilt  = 9;
     22
     23  if (!dvo_catalog_open (&catalog, NULL, VERBOSE, "r")) { // *** FUNC
     24    fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename);
     25    exit (1);
     26  }
     27  if (!catalog.Naverage_disk) {
     28    if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename);
    9429    dvo_catalog_unlock (&catalog);
    9530    dvo_catalog_free (&catalog);
    96 
    97     if (!status) {
    98       fprintf (stderr, "failure to insert data in mysql\n");
    99       exit (2);
    100     }
    101   }
    102 
    103   if (SAVE_REMOTE && HOST_ID) {
    104     if (!save_detections_dvopsps ()) {
    105       fprintf (stderr, "ERROR: failure to save output file with detections\n");
    106       status = FALSE;
    107     }
    108   }
     31    FREE (localFilename);
     32    exit (1);
     33  }
     34
     35  if (!append_detections_dvopsps_catalog (&catalog)) { // *** FUNC : OK
     36    fprintf (stderr, "ERROR: failure to append detections to output catalog\n");
     37    status = FALSE;
     38  }
     39   
     40  dvo_catalog_unlock (&catalog); // *** FUNC
     41  dvo_catalog_free (&catalog); // *** FUNC
     42  FREE (localFilename);
     43
     44  if (!status) {
     45    fprintf (stderr, "failure to insert data in mysql\n");
     46    exit (2);
     47  }
     48
     49  if (!save_detections_dvopsps ()) { // *** FUNC : OK
     50    fprintf (stderr, "ERROR: failure to save output file with detections\n");
     51    status = FALSE;
     52  }
     53
     54  exit (1);
    10955
    11056  return (status);
    … …  
    11864  HostTable *table = HostTableLoad (CATDIR, sky->hosts);
    11965
    120   char uniquer[12];
     66  char uniquer[16];
    12167  int TIME = time(NULL);
    12268  int PID = getpid();

  • branches/eam_branches/ohana.20170809/src/dvopsps/src/insert_detections_dvopsps_catalog.c

    r39645 r40114  
    22# define DEBUG 0
    33
    4 int Ndetections = 0;
    5 int NDETECTIONS = 0;
    6 Detections *detections = NULL;
     4static int Ndetections = 0;
     5static int NDETECTIONS = 0;
     6static Detections *detections = NULL;
    77
    88static float ZeroPoint = 25.0;
    … …  
    1818  off_t i, j;
    1919
    20   ZeroPoint = GetZeroPoint();
     20  ZeroPoint = 25.0;
    2121
    2222  int missingID = 0;
    … …  
    3131  if (TIME_START) ohana_str_to_time (TIME_START, &TIME_START_SEC); // we validate this in the args.c
    3232  if (TIME_END)   ohana_str_to_time (TIME_END,   &TIME_END_SEC);   // we validate this in the args.c
     33  // XXX the above ohana_str_to_time are NOT called
    3334
    3435  for (i = 0; i < catalog[0].Naverage; i++) {
    … …  
    4142    for (j = 0; j < catalog[0].average[i].Nmeasure; j++) {
    4243
     44      myAssert (m + j < catalog[0].Nmeasure, "out of bounds");
     45      myAssert (m + j >= 0, "out of bounds");
     46
    4347      Average *average = &catalog->average[i];
    4448      Measure *measure = &catalog->measure[m + j];
    4549
    46       // some filters -- these are the detections we skip
    47       if (TIME_START && (measure->t < TIME_START_SEC)) continue;
    48       if (TIME_END   && (measure->t >=  TIME_END_SEC)) continue;
    49 
    50       if (measure->photcode < PHOTCODE_START) continue;
    51       if (measure->photcode >=  PHOTCODE_END) continue;
    52 
    53       assign_detection_values (&detections[Ndetections], measure, average);
     50      if (measure->photcode < 12000) continue;
     51
     52      assign_detection_values (&detections[Ndetections], measure, average); // *** FUNC : OK
    5453      Ndetections ++;
    5554
    … …  
    5857  }
    5958
    60   fprintf (stderr, "write %d detections, %d missing extID\n", Ndetections, missingID);
     59  fprintf (stderr, "write %d of %d detections, %d missing extID\n", Ndetections, NDETECTIONS, missingID);
    6160  return TRUE;
    6261}
    … …  
    7069  int status = TRUE;
    7170
    72   if (!DetectionsSave (RESULT_FILE, detections, Ndetections)) {
     71//if (!DetectionsSave (RESULT_FILE, detections, Ndetections)) {
     72  if (!DetectionsSave ("/data/ipp110.0/eugene/3pi.dvo.catdirs/3pi.pv3.20170216/catdir.037/dvopsps.09762.24244.det.dat", detections, Ndetections)) {
    7373    fprintf (stderr, "failed to save detection file %s\n", RESULT_FILE);
    7474    status = FALSE;
    … …  
    305305int assign_detection_values (Detections *detection, Measure *measure, Average *average) {
    306306
     307  return TRUE;
     308
    307309  PhotCode *code = GetPhotcodebyCode(measure->photcode);
     310  myAssert (code, "oops");
    308311
    309312  uint64_t ippObjID = ((uint64_t)average->catID << 32) + (uint64_t)average->objID; // ippObjID
Note: See TracChangeset for help on using the changeset viewer.