Index: /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/include/dvopsps.h =================================================================== --- /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/include/dvopsps.h (revision 35092) +++ /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/include/dvopsps.h (revision 35093) @@ -71,5 +71,5 @@ int insert_detections_mysql_init PROTO((IOBuffer *buffer)); -int insert_detections_mysql_array PROTO((Detections *detections, int Ndetections)); +int insert_detections_mysql_array PROTO((MYSQL *mysql, Detections *detections, int Ndetections)); int insert_detections_mysql_detvalue PROTO((IOBuffer *buffer, Detections *detection)); @@ -79,5 +79,5 @@ int insert_objects_dvopsps PROTO(()); -int insert_objects_dvopsps_parallel PROTO((SkyTable *sky)); +int insert_objects_dvopsps_parallel PROTO((SkyList *sky)); int insert_objects_dvopsps_catalog PROTO((Catalog *catalog, char *basename, MYSQL *mysql)); Index: /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/DetectionOps.c =================================================================== --- /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/DetectionOps.c (revision 35092) +++ /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/DetectionOps.c (revision 35093) @@ -148,7 +148,10 @@ gfits_define_bintable_column (&theader, "J", "imageID", NULL, NULL, 1.0, BZERO_INT32); gfits_define_bintable_column (&theader, "J", "ippDetectID", NULL, NULL, 1.0, BZERO_INT32); - gfits_define_bintable_column (&theader, "K", "detectID", NULL, NULL, 1.0, BZERO_INT64); - gfits_define_bintable_column (&theader, "K", "ippObjID", NULL, NULL, 1.0, BZERO_INT64); - gfits_define_bintable_column (&theader, "K", "objID", NULL, NULL, 1.0, BZERO_INT64); + gfits_define_bintable_column (&theader, "K", "detectID", NULL, NULL, 1.0, 0); + gfits_define_bintable_column (&theader, "K", "ippObjID", NULL, NULL, 1.0, 0); + gfits_define_bintable_column (&theader, "K", "objID", NULL, NULL, 1.0, 0); + // gfits_define_bintable_column (&theader, "K", "detectID", NULL, NULL, 1.0, BZERO_INT64); + // gfits_define_bintable_column (&theader, "K", "ippObjID", NULL, NULL, 1.0, BZERO_INT64); + // gfits_define_bintable_column (&theader, "K", "objID", NULL, NULL, 1.0, BZERO_INT64); gfits_define_bintable_column (&theader, "J", "flags", NULL, NULL, 1.0, BZERO_INT32); gfits_define_bintable_column (&theader, "E", "zp", NULL, NULL, 1.0, 0.0); Index: /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/dvopsps_client.c =================================================================== --- /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/dvopsps_client.c (revision 35092) +++ /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/dvopsps_client.c (revision 35093) @@ -16,5 +16,14 @@ initialize_dvopsps_client (argc, argv); - status = insert_detections_dvopsps (); + if (!strcasecmp (argv[1], "detections")) { + status = insert_detections_dvopsps (); + } + if (!strcasecmp (argv[1], "objects")) { + status = insert_objects_dvopsps (); + } + if (status == -1) { + fprintf (stderr, "invalid mode, should be : detections, skytable\n"); + exit (2); + } if (!status) exit (1); Index: /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/initialize_dvopsps.c =================================================================== --- /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/initialize_dvopsps.c (revision 35092) +++ /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/initialize_dvopsps.c (revision 35093) @@ -107,4 +107,5 @@ } + HOST_ID = 0; PARALLEL = FALSE; if ((N = get_argument (argc, argv, "-parallel"))) { @@ -193,4 +194,9 @@ int N; + // by definition, the client is not parallel + PARALLEL = FALSE; + PARALLEL_MANUAL = FALSE; + PARALLEL_SERIAL = FALSE; + if ((N = get_argument (argc, argv, "-dbhost"))) { remove_argument (N, &argc, argv); Index: /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_detections_dvopsps.c =================================================================== --- /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_detections_dvopsps.c (revision 35092) +++ /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_detections_dvopsps.c (revision 35093) @@ -42,5 +42,5 @@ MYSQL mysqlBase; MYSQL *mysqlReal = NULL; - if (!PARALLEL || !SAVE_REMOTE) { + if (!HOST_ID || !SAVE_REMOTE) { mysqlReal = mysql_dvopsps_connect (&mysqlBase); if (!mysqlReal) { @@ -174,4 +174,13 @@ Detections *detections = NULL; + // NOTE: mysql connection happens here since each dvopsps_client makes its own connection + MYSQL mysqlBase; + MYSQL *mysql = NULL; + mysql = mysql_dvopsps_connect (&mysqlBase); + if (!mysql) { + fprintf (stderr, "failed to connect to mysql\n"); + exit (1); + } + for (i = 0; i < table->Nhosts; i++) { while ((detections = DetectionsLoad (table->hosts[i].results, &Ndetections)) == NULL) { @@ -183,5 +192,5 @@ table->hosts[i].results = NULL; - insert_detections_mysql_array (detections, Ndetections); + insert_detections_mysql_array (mysql, detections, Ndetections); free (detections); @@ -192,5 +201,5 @@ } -int insert_detections_mysql_array (Detections *detections, int Ndetections) { +int insert_detections_mysql_array (MYSQL *mysql, Detections *detections, int Ndetections) { int i; @@ -201,13 +210,4 @@ if (Ndetections == 0) return TRUE; - - // NOTE: mysql connection happens here since each dvopsps_client makes its own connection - MYSQL mysqlBase; - MYSQL *mysql = NULL; - mysql = mysql_dvopsps_connect (&mysqlBase); - if (!mysql) { - fprintf (stderr, "failed to connect to mysql\n"); - exit (1); - } insert_detections_mysql_init (&buffer); Index: /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c =================================================================== --- /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c (revision 35092) +++ /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c (revision 35093) @@ -55,5 +55,5 @@ detections[Ndetections].expTime = pow(10.0, 0.4 * measure->dt); // expTime detections[Ndetections].ra = average->R - measure->dR / 3600.; // ra - detections[Ndetections].dec = average->D - measure->dR / 3600.; // dec + detections[Ndetections].dec = average->D - measure->dD / 3600.; // dec detections[Ndetections].raErr = measure->dXccd * 0.01 * fabs(measure->pltscale); // estimate of raErr detections[Ndetections].decErr = measure->dYccd * 0.01 * fabs(measure->pltscale); // estimate of decErr Index: /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_objects_dvopsps.c =================================================================== --- /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_objects_dvopsps.c (revision 35092) +++ /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_objects_dvopsps.c (revision 35093) @@ -14,9 +14,4 @@ SkyTableSetFilenames (sky, CATDIR, "cpt"); - if (PARALLEL && !HOST_ID) { - int status = insert_objects_dvopsps_parallel (sky); - return status; - } - // determine the populated SkyRegions overlapping the requested area (default depth) if (SINGLE_CPT) { @@ -26,4 +21,10 @@ } myAssert (skylist, "ooops!"); + + // pass the skylist to the parallel version, and only launch on relevant hosts + if (PARALLEL && !HOST_ID) { + int status = insert_objects_dvopsps_parallel (skylist); + return status; + } # if (USE_MYSQL) @@ -93,5 +94,5 @@ # define DEBUG 1 -int insert_objects_dvopsps_parallel (SkyTable *sky) { +int insert_objects_dvopsps_parallel (SkyList *sky) { // launch the dvopsps_client jobs to the parallel hosts @@ -100,6 +101,23 @@ HostTable *table = HostTableLoad (CATDIR, sky->hosts); - int i; + // if sky->Nregions < 0.5*Nhosts, check if the host used before launching... + + int i, j; for (i = 0; i < table->Nhosts; i++) { + + if (sky->Nregions < table->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname); + continue; + } + } // ensure that the paths are absolute path names @@ -113,6 +131,4 @@ UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); - fprintf (stderr, "command: %s\n", command); - char tmpline[1024]; snprintf (tmpline, 1024, "%s -dbhost %s", command, DATABASE_HOST); strcpy (command, tmpline); @@ -121,5 +137,8 @@ snprintf (tmpline, 1024, "%s -dbname %s", command, DATABASE_NAME); strcpy (command, tmpline); - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } + if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } + if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } + + fprintf (stderr, "command: %s\n", command); if (PARALLEL_MANUAL) continue; Index: /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_objects_dvopsps_catalog.c =================================================================== --- /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_objects_dvopsps_catalog.c (revision 35092) +++ /branches/eam_branches/ipp-20121219/Ohana/src/dvopsps/src/insert_objects_dvopsps_catalog.c (revision 35093) @@ -68,10 +68,10 @@ } insert_objects_mysql_commit (&ave_buffer, &sec_buffer, mysql); - if (VERBOSE) fprintf (stderr, "inserted %d rows\n", Ninsert); + if (VERBOSE) fprintf (stderr, "inserted "OFF_T_FMT" average, "OFF_T_FMT" secfilt\n", catalog[0].Naverage, Nsecfilt*catalog[0].Naverage); FreeIOBuffer (&ave_buffer); FreeIOBuffer (&sec_buffer); free (cleanname); - MARKTIME("-- inserted "OFF_T_FMT" rows in %f sec, skipped %d without IDs\n", found, dtime, missingID); + MARKTIME("-- inserted "OFF_T_FMT" objects in %f sec, skipped %d without IDs\n", found, dtime, missingID); return (TRUE); } @@ -82,5 +82,14 @@ IOBuffer buffer; InitIOBuffer (&buffer, 1024); - MYSQL_RES *result; + // MYSQL_RES *result; + + PrintIOBuffer (&buffer, "DROP TABLE %s_cpt\n", basename); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to drop table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + } + buffer.Nbuffer = 0; + bzero (buffer.buffer, buffer.Nalloc); PrintIOBuffer (&buffer, "CREATE TABLE %s_cpt (" @@ -121,11 +130,20 @@ status = mysql_query(mysql, buffer.buffer); if (status) { - fprintf (stderr, "error with insert:\n"); + fprintf (stderr, "failed to create table:\n"); fprintf (stderr, "%s\n", mysql_error(mysql)); fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer); } - result = mysql_store_result (mysql); - mysql_free_result (result); - + // result = mysql_store_result (mysql); + // mysql_free_result (result); + + buffer.Nbuffer = 0; + bzero (buffer.buffer, buffer.Nalloc); + + PrintIOBuffer (&buffer, "DROP TABLE %s_cps\n", basename); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to drop table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + } buffer.Nbuffer = 0; bzero (buffer.buffer, buffer.Nalloc); @@ -155,10 +173,10 @@ status = mysql_query(mysql, buffer.buffer); if (status) { - fprintf (stderr, "error with insert:\n"); + fprintf (stderr, "failed to create table:\n"); fprintf (stderr, "%s\n", mysql_error(mysql)); fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer); } - result = mysql_store_result (mysql); - mysql_free_result (result); + // result = mysql_store_result (mysql); + // mysql_free_result (result); FreeIOBuffer (&buffer); @@ -330,5 +348,5 @@ } result = mysql_store_result (mysql); - mysql_free_result (result); + if (result) mysql_free_result (result); if (DEBUG) fprintf (stderr, "%s\n", sec_buffer->buffer); @@ -340,5 +358,5 @@ } result = mysql_store_result (mysql); - mysql_free_result (result); + if (result) mysql_free_result (result); } else { fprintf (stderr, "%s\n", sec_buffer->buffer);