IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 35093


Ignore:
Timestamp:
Feb 6, 2013, 11:17:39 AM (13 years ago)
Author:
eugene
Message:

when using master mysql I/O for detections, only connect to mysql on the master (and only connect once); do not use BZERO/BSCALE for int64 fields; drop/create object tables (and do not call mysql_store_result for create/drop calls); only launch dvopsps_client for hosts actually needed for selected regions; pass -cpt option to client if supplied to master

Location:
branches/eam_branches/ipp-20121219/Ohana/src/dvopsps
Files:
8 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/include/dvopsps.h

    r35069 r35093  
    7171int    insert_detections_mysql_init       PROTO((IOBuffer *buffer));
    7272
    73 int    insert_detections_mysql_array      PROTO((Detections *detections, int Ndetections));
     73int    insert_detections_mysql_array      PROTO((MYSQL *mysql, Detections *detections, int Ndetections));
    7474int    insert_detections_mysql_detvalue   PROTO((IOBuffer *buffer, Detections *detection));
    7575
    … …  
    7979
    8080int    insert_objects_dvopsps             PROTO(());
    81 int    insert_objects_dvopsps_parallel    PROTO((SkyTable *sky));
     81int    insert_objects_dvopsps_parallel    PROTO((SkyList *sky));
    8282int    insert_objects_dvopsps_catalog     PROTO((Catalog *catalog, char *basename, MYSQL *mysql));
    8383                                         

  • branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/DetectionOps.c

    r35069 r35093  
    148148  gfits_define_bintable_column (&theader, "J", "imageID",     NULL, NULL, 1.0, BZERO_INT32);
    149149  gfits_define_bintable_column (&theader, "J", "ippDetectID", NULL, NULL, 1.0, BZERO_INT32);
    150   gfits_define_bintable_column (&theader, "K", "detectID",    NULL, NULL, 1.0, BZERO_INT64);
    151   gfits_define_bintable_column (&theader, "K", "ippObjID",    NULL, NULL, 1.0, BZERO_INT64);
    152   gfits_define_bintable_column (&theader, "K", "objID",       NULL, NULL, 1.0, BZERO_INT64);
     150  gfits_define_bintable_column (&theader, "K", "detectID",    NULL, NULL, 1.0, 0);
     151  gfits_define_bintable_column (&theader, "K", "ippObjID",    NULL, NULL, 1.0, 0);
     152  gfits_define_bintable_column (&theader, "K", "objID",       NULL, NULL, 1.0, 0);
     153  // gfits_define_bintable_column (&theader, "K", "detectID",    NULL, NULL, 1.0, BZERO_INT64);
     154  // gfits_define_bintable_column (&theader, "K", "ippObjID",    NULL, NULL, 1.0, BZERO_INT64);
     155  // gfits_define_bintable_column (&theader, "K", "objID",       NULL, NULL, 1.0, BZERO_INT64);
    153156  gfits_define_bintable_column (&theader, "J", "flags",       NULL, NULL, 1.0, BZERO_INT32);
    154157  gfits_define_bintable_column (&theader, "E", "zp",          NULL, NULL, 1.0, 0.0);

  • branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/dvopsps_client.c

    r34916 r35093  
    1616  initialize_dvopsps_client (argc, argv);
    1717
    18   status = insert_detections_dvopsps ();
     18  if (!strcasecmp (argv[1], "detections")) {
     19    status = insert_detections_dvopsps ();
     20  }
     21  if (!strcasecmp (argv[1], "objects")) {
     22    status = insert_objects_dvopsps ();
     23  }
     24  if (status == -1) {
     25    fprintf (stderr, "invalid mode, should be : detections, skytable\n");
     26    exit (2);
     27  }
    1928
    2029  if (!status) exit (1);

  • branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/initialize_dvopsps.c

    r35069 r35093  
    107107  }
    108108
     109  HOST_ID = 0;
    109110  PARALLEL = FALSE;
    110111  if ((N = get_argument (argc, argv, "-parallel"))) {
    … …  
    193194  int N;
    194195
     196  // by definition, the client is not parallel
     197  PARALLEL = FALSE;
     198  PARALLEL_MANUAL = FALSE;
     199  PARALLEL_SERIAL = FALSE;
     200
    195201  if ((N = get_argument (argc, argv, "-dbhost"))) {
    196202    remove_argument (N, &argc, argv);

  • branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_detections_dvopsps.c

    r35078 r35093  
    4242  MYSQL  mysqlBase;
    4343  MYSQL *mysqlReal = NULL;
    44   if (!PARALLEL || !SAVE_REMOTE) {
     44  if (!HOST_ID || !SAVE_REMOTE) {
    4545    mysqlReal = mysql_dvopsps_connect (&mysqlBase);
    4646    if (!mysqlReal) {
    … …  
    174174    Detections *detections = NULL;
    175175
     176    // NOTE: mysql connection happens here since each dvopsps_client makes its own connection
     177    MYSQL  mysqlBase;
     178    MYSQL *mysql = NULL;
     179    mysql = mysql_dvopsps_connect (&mysqlBase);
     180    if (!mysql) {
     181      fprintf (stderr, "failed to connect to mysql\n");
     182      exit (1);
     183    }
     184
    176185    for (i = 0; i < table->Nhosts; i++) {
    177186      while ((detections = DetectionsLoad (table->hosts[i].results, &Ndetections)) == NULL) {
    … …  
    183192      table->hosts[i].results = NULL;
    184193   
    185       insert_detections_mysql_array (detections, Ndetections);
     194      insert_detections_mysql_array (mysql, detections, Ndetections);
    186195
    187196      free (detections);
    … …  
    192201}     
    193202
    194 int insert_detections_mysql_array (Detections *detections, int Ndetections) {
     203int insert_detections_mysql_array (MYSQL *mysql, Detections *detections, int Ndetections) {
    195204
    196205  int i;
    … …  
    201210
    202211  if (Ndetections == 0) return TRUE;
    203 
    204   // NOTE: mysql connection happens here since each dvopsps_client makes its own connection
    205   MYSQL  mysqlBase;
    206   MYSQL *mysql = NULL;
    207   mysql = mysql_dvopsps_connect (&mysqlBase);
    208   if (!mysql) {
    209     fprintf (stderr, "failed to connect to mysql\n");
    210     exit (1);
    211   }
    212212
    213213  insert_detections_mysql_init (&buffer);

  • branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c

    r35070 r35093  
    5555      detections[Ndetections].expTime      = pow(10.0, 0.4 * measure->dt);    // expTime
    5656      detections[Ndetections].ra           = average->R - measure->dR / 3600.;    // ra
    57       detections[Ndetections].dec          = average->D - measure->dR / 3600.;    // dec
     57      detections[Ndetections].dec          = average->D - measure->dD / 3600.;    // dec
    5858      detections[Ndetections].raErr        = measure->dXccd * 0.01 * fabs(measure->pltscale); // estimate of raErr
    5959      detections[Ndetections].decErr       = measure->dYccd * 0.01 * fabs(measure->pltscale); // estimate of decErr

  • branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_objects_dvopsps.c

    r35055 r35093  
    1414  SkyTableSetFilenames (sky, CATDIR, "cpt");
    1515 
    16   if (PARALLEL && !HOST_ID) {
    17     int status = insert_objects_dvopsps_parallel (sky);
    18     return status;
    19   }
    20 
    2116  // determine the populated SkyRegions overlapping the requested area (default depth)
    2217  if (SINGLE_CPT) {
    … …  
    2621  }
    2722  myAssert (skylist, "ooops!");
     23
     24  // pass the skylist to the parallel version, and only launch on relevant hosts
     25  if (PARALLEL && !HOST_ID) {
     26    int status = insert_objects_dvopsps_parallel (skylist);
     27    return status;
     28  }
    2829
    2930# if (USE_MYSQL)
    … …  
    9394# define DEBUG 1
    9495
    95 int insert_objects_dvopsps_parallel (SkyTable *sky) {
     96int insert_objects_dvopsps_parallel (SkyList *sky) {
    9697
    9798  // launch the dvopsps_client jobs to the parallel hosts
    … …  
    100101  HostTable *table = HostTableLoad (CATDIR, sky->hosts);
    101102
    102   int i;
     103  // if sky->Nregions < 0.5*Nhosts, check if the host used before launching...
     104
     105  int i, j;
    103106  for (i = 0; i < table->Nhosts; i++) {
     107
     108    if (sky->Nregions < table->Nhosts) {
     109      // do any of the regions want this host?
     110      int wantThisHost = FALSE;
     111      for (j = 0; j < sky->Nregions; j++) {
     112        if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) {
     113          wantThisHost = TRUE;
     114          break;
     115        }
     116      }
     117      if (!wantThisHost) {
     118        // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname);
     119        continue;
     120      }
     121    }
    104122
    105123    // ensure that the paths are absolute path names
    … …  
    113131              UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
    114132
    115     fprintf (stderr, "command: %s\n", command);
    116 
    117133    char tmpline[1024];
    118134    snprintf (tmpline, 1024, "%s -dbhost %s", command, DATABASE_HOST); strcpy (command, tmpline);
    … …  
    121137    snprintf (tmpline, 1024, "%s -dbname %s", command, DATABASE_NAME); strcpy (command, tmpline);
    122138
    123     if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); }
     139    if (VERBOSE)    { snprintf (tmpline, 1024, "%s -v",      command);             strcpy (command, tmpline); }
     140    if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); }
     141
     142    fprintf (stderr, "command: %s\n", command);
    124143
    125144    if (PARALLEL_MANUAL) continue;

  • branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_objects_dvopsps_catalog.c

    r35055 r35093  
    6868  }
    6969  insert_objects_mysql_commit (&ave_buffer, &sec_buffer, mysql);
    70   if (VERBOSE) fprintf (stderr, "inserted %d rows\n", Ninsert);
     70  if (VERBOSE) fprintf (stderr, "inserted "OFF_T_FMT" average, "OFF_T_FMT" secfilt\n", catalog[0].Naverage, Nsecfilt*catalog[0].Naverage);
    7171  FreeIOBuffer (&ave_buffer);
    7272  FreeIOBuffer (&sec_buffer);
    7373  free (cleanname);
    7474
    75   MARKTIME("-- inserted "OFF_T_FMT" rows in %f sec, skipped %d without IDs\n", found, dtime, missingID);
     75  MARKTIME("-- inserted "OFF_T_FMT" objects in %f sec, skipped %d without IDs\n", found, dtime, missingID);
    7676  return (TRUE);
    7777}
    … …  
    8282  IOBuffer buffer;
    8383  InitIOBuffer (&buffer, 1024);
    84   MYSQL_RES *result;
     84  // MYSQL_RES *result;
     85
     86  PrintIOBuffer (&buffer, "DROP TABLE %s_cpt\n", basename);
     87  status = mysql_query(mysql, buffer.buffer);
     88  if (status) {
     89    fprintf (stderr, "failed to drop table:\n");
     90    fprintf (stderr, "%s\n", mysql_error(mysql));
     91  }
     92  buffer.Nbuffer = 0;
     93  bzero (buffer.buffer, buffer.Nalloc);
    8594
    8695  PrintIOBuffer (&buffer, "CREATE TABLE %s_cpt ("
    … …  
    121130  status = mysql_query(mysql, buffer.buffer);
    122131  if (status) {
    123     fprintf (stderr, "error with insert:\n");
     132    fprintf (stderr, "failed to create table:\n");
    124133    fprintf (stderr, "%s\n", mysql_error(mysql));
    125134    fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer);
    126135  }
    127   result = mysql_store_result (mysql);
    128   mysql_free_result (result);
    129 
     136  // result = mysql_store_result (mysql);
     137  // mysql_free_result (result);
     138
     139  buffer.Nbuffer = 0;
     140  bzero (buffer.buffer, buffer.Nalloc);
     141
     142  PrintIOBuffer (&buffer, "DROP TABLE %s_cps\n", basename);
     143  status = mysql_query(mysql, buffer.buffer);
     144  if (status) {
     145    fprintf (stderr, "failed to drop table:\n");
     146    fprintf (stderr, "%s\n", mysql_error(mysql));
     147  }
    130148  buffer.Nbuffer = 0;
    131149  bzero (buffer.buffer, buffer.Nalloc);
    … …  
    155173  status = mysql_query(mysql, buffer.buffer);
    156174  if (status) {
    157     fprintf (stderr, "error with insert:\n");
     175    fprintf (stderr, "failed to create table:\n");
    158176    fprintf (stderr, "%s\n", mysql_error(mysql));
    159177    fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer);
    160178  }
    161   result = mysql_store_result (mysql);
    162   mysql_free_result (result);
     179  // result = mysql_store_result (mysql);
     180  // mysql_free_result (result);
    163181
    164182  FreeIOBuffer (&buffer);
    … …  
    330348    }
    331349    result = mysql_store_result (mysql);
    332     mysql_free_result (result);
     350    if (result) mysql_free_result (result);
    333351
    334352    if (DEBUG) fprintf (stderr, "%s\n", sec_buffer->buffer);
    … …  
    340358    }
    341359    result = mysql_store_result (mysql);
    342     mysql_free_result (result);
     360    if (result) mysql_free_result (result);
    343361  } else {
    344362    fprintf (stderr, "%s\n", sec_buffer->buffer);
Note: See TracChangeset for help on using the changeset viewer.