IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 37938 for trunk/ippTools


Ignore:
Timestamp:
Feb 26, 2015, 3:53:04 PM (11 years ago)
Author:
watersc1
Message:

Bringing back old method of queuing warp-stack diffs. This seems to do the right thing and not construct partial diffs for warps missing stacks to diff against.

Location:
trunk/ippTools/src
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTools/src/difftool.c

    r37900 r37938  
    4242static bool definepoprunMode(pxConfig *config);
    4343static bool definewarpstackMode(pxConfig *config);
     44static bool definewarpstackOldMethodMode(pxConfig *config);
    4445static bool definewarpwarpMode(pxConfig *config);
    4546static bool definestackstackMode(pxConfig *config);
    … …  
    9596        MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
    9697        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,       definewarpstackMode);
     98        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACKOLDMETHOD,       definewarpstackOldMethodMode);
    9799        MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
    98100        MODECASE(DIFFTOOL_MODE_DEFINESTACKSTACK,      definestackstackMode);
    … …  
    12501252  }
    12511253 
    1252  
     1254  PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
    12531255  PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
    12541256 
    … …  
    13341336    return true;
    13351337  }
     1338
     1339  // Do a pass through the result list to determine which runs should be excluded.
     1340  // This exclusion should do two things:  If a skycell doesn't have a stack (as we do a LEFT JOIN) now,
     1341  // then that skycell needs to not be included in the diff.  However, if we have available off,
     1342  // then we need to flag the entire exposure as bad.
     1343  psS64 previous_exp_id = 0;
     1344  long  start_index     = 0;
     1345  bool  bad_exposure    = false;
     1346  long  Nskycells       = 0;
     1347  for (long i = 0; i < output->n; i++) {
     1348    psMetadata *row = output->data[i];
     1349    bool mdok;
     1350    psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
     1351    if (!mdok) {
     1352        psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
     1353        if (!psDBRollback(config->dbh)) {
     1354          psError(PS_ERR_UNKNOWN, false, "database error");
     1355        }
     1356        return false;
     1357    }
     1358
     1359    if (exp_id != previous_exp_id) { // Save the position of the start of this exposure.
     1360      if (Nskycells == 0) {  // Ooops, we didn't find any good skycells for this exposure.  Don't make an empty run.
     1361        for (long j = start_index; j < i; j++) { // Clear all the previous entries for this exposure.
     1362          psMetadata *old_row = output->data[j];
     1363          psMetadataAddBool(old_row,PS_LIST_TAIL, "exclude", PS_META_REPLACE, "", true);  // No stack exists, so this row can't be diffed.
     1364          psMetadataAddBool(old_row,PS_LIST_TAIL, "excludeExp", PS_META_REPLACE, "", true);  // No stack exists, so this row can't be diffed.
     1365        }
     1366      }     
     1367      previous_exp_id = exp_id;
     1368      start_index = i;
     1369      bad_exposure = false;
     1370      Nskycells = 0;
     1371    }
     1372
     1373    if (bad_exposure) { // We've flagged this exposure as bad.
     1374      psMetadataAddBool(row,PS_LIST_TAIL, "exclude", PS_META_REPLACE, "", true);
     1375      psMetadataAddBool(row,PS_LIST_TAIL, "excludeExp", PS_META_REPLACE, "", true);
     1376      continue;
     1377    }
     1378
     1379    psMetadataAddBool(row,PS_LIST_TAIL, "exclude", PS_META_REPLACE, "", false); // By default, this shouldn't be excluded.
     1380    psMetadataAddBool(row,PS_LIST_TAIL, "excludeExp", PS_META_REPLACE, "", false);
     1381   
     1382    psS64 stack_id = psMetadataLookupS64(&mdok, row, "stack_id");
     1383    if (stack_id == PS_MAX_S64) { // stack_id IS NULL, we cannot make this skycell-level diff
     1384      psMetadataAddBool(row,PS_LIST_TAIL, "exclude", PS_META_REPLACE, "", true);  // No stack exists, so this row can't be diffed.
     1385
     1386      if (!available) { // We want to be strict about everything, so cancel this entire exposure.
     1387        bad_exposure = true;  // New exposures with this exp_id will be flagged as bad.
     1388        for (long j = start_index; j < i; j++) { // Clear all the previous entries for this exposure.
     1389          psMetadata *old_row = output->data[j];
     1390          psMetadataAddBool(old_row,PS_LIST_TAIL, "exclude", PS_META_REPLACE, "", true);  // No stack exists, so this row can't be diffed.
     1391          psMetadataAddBool(old_row,PS_LIST_TAIL, "excludeExp", PS_META_REPLACE, "", true);  // No stack exists, so this row can't be diffed.
     1392        }
     1393      }
     1394    }
     1395    else { // This looks fine.
     1396      Nskycells++;
     1397    }
     1398  }
     1399 
     1400
     1401 
    13361402  if (pretend) {
    13371403    // negative simple so the default is true
    … …  
    13851451          return false;
    13861452        }
     1453      }
     1454
     1455      // Do exposure level exclusion here.
     1456      bool excludeExp = psMetadataLookupBool(&mdok, row, "excludeExp");
     1457      if (!mdok) {
     1458        psError(PXTOOLS_ERR_PROG, false, "excludeExp not found");
     1459        if (!psDBRollback(config->dbh)) {
     1460          psError(PS_ERR_UNKNOWN, false, "database error");
     1461        }
     1462        return false;
     1463      }
     1464      if (excludeExp) { // The pre-scan determined that no diff should be made for this exposure.
     1465        continue;
    13871466      }
    13881467     
    … …  
    14411520   
    14421521    if (exp_id == last_exp_id) {
     1522      bool exclude = psMetadataLookupBool(&mdok, row, "exclude");
     1523      if (!mdok) {
     1524        psError(PXTOOLS_ERR_PROG, false, "exclude not found");
     1525        if (!psDBRollback(config->dbh)) {
     1526          psError(PS_ERR_UNKNOWN, false, "database error");
     1527        }
     1528        return false;
     1529      }
     1530      if (exclude) { // skip this skycell, as we probably don't have a diff.
     1531        continue;
     1532      }
    14431533      psString skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id");
    14441534      if (!mdok) {
    … …  
    15241614} 
    15251615
    1526 #if (0)
    1527 static bool definewarpstackMode(pxConfig *config)
     1616
     1617static bool definewarpstackOldMethodMode(pxConfig *config)
    15281618{
    15291619    PS_ASSERT_PTR_NON_NULL(config, false);
    … …  
    18061896        }
    18071897
     1898        // CZW force available off.  If we're using this very-expensive mode, we need to get the one thing it does correctly.
     1899        available = false;;
    18081900        if (!available && (num != skycell_count)) {
    18091901            psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" PRId64
    … …  
    19132005    return true;
    19142006}
    1915 #endif
     2007
    19162008
    19172009static bool definewarpwarpMode(pxConfig *config)

  • trunk/ippTools/src/difftool.h

    r29416 r37938  
    5454    DIFFTOOL_MODE_LISTSSRUN,
    5555    DIFFTOOL_MODE_SETSKYFILETOUPDATE,
     56    DIFFTOOL_MODE_DEFINEWARPSTACKOLDMETHOD,
    5657} difftoolMode;
    5758

  • trunk/ippTools/src/difftoolConfig.c

    r37900 r37938  
    273273
    274274    pxspaceBoxAddArguments(definewarpstackArgs);
     275
     276   
     277    // -definewarpstackOldMethod
     278    psMetadata *definewarpstackOldMethodArgs = psMetadataAlloc();
     279    psMetadataAddS64(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
     280    psMetadataAddS64(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
     281    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL);
     282    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL);
     283    psMetadataAddBool(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-bothways",  0,          "do the subtraction both ways?", false);
     284    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
     285    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-comment", 0, "search by comment (LIKE)", NULL);
     286    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL);
     287    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-stack_data_group", 0, "search by stack data_group", NULL);
     288    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
     289    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL);
     290    psMetadataAddF32(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
     291    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL);
     292    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-set_label",  0, "define label", NULL);
     293    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-set_reduction",  0, "define reduction class", NULL);
     294    psMetadataAddTime(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now);
     295    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-set_data_group",  0,     "define data group", NULL);
     296    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-set_dist_group",  0,     "define dist group", NULL);
     297    psMetadataAddStr(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-set_note",  0,           "define note", NULL);
     298    psMetadataAddBool(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false);
     299    psMetadataAddBool(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false);
     300    psMetadataAddBool(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-available", 0, "define new run even if no stacks available for some skycells", false);
     301    psMetadataAddBool(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     302    psMetadataAddBool(definewarpstackOldMethodArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false);
     303
     304    pxspaceBoxAddArguments(definewarpstackOldMethodArgs);
    275305
    276306   
    … …  
    481511    PXOPT_ADD_MODE("-definepoprun",     "", DIFFTOOL_MODE_DEFINEPOPRUN,      definepoprunArgs);
    482512    PXOPT_ADD_MODE("-definewarpstack",  "", DIFFTOOL_MODE_DEFINEWARPSTACK,   definewarpstackArgs);
     513    PXOPT_ADD_MODE("-definewarpstackOldMethod",  "", DIFFTOOL_MODE_DEFINEWARPSTACKOLDMETHOD,   definewarpstackOldMethodArgs);
    483514    PXOPT_ADD_MODE("-definewarpwarp",   "", DIFFTOOL_MODE_DEFINEWARPWARP,    definewarpwarpArgs);
    484515    PXOPT_ADD_MODE("-definestackstack", "", DIFFTOOL_MODE_DEFINESTACKSTACK,  definestackstackArgs);
Note: See TracChangeset for help on using the changeset viewer.