Index: trunk/ippTools/src/bgtool.c
===================================================================
--- trunk/ippTools/src/bgtool.c	(revision 29493)
+++ trunk/ippTools/src/bgtool.c	(revision 29671)
@@ -49,12 +49,20 @@
 static bool revertwarpMode(pxConfig *config);
 static bool listwarpMode(pxConfig *config);
-static bool tocleanchipMode(pxConfig *config);
-static bool cleanedchipMode(pxConfig *config);
-static bool tocleanwarpMode(pxConfig *config);
-static bool cleanedwarpMode(pxConfig *config);
+
+static bool pendingcleanupchiprunMode(pxConfig *config);
+static bool pendingcleanupchipimfileMode(pxConfig *config);
+static bool tocleanedchipimfileMode(pxConfig *config);
+static bool updatechipimfileMode(pxConfig *config);
+
+static bool pendingcleanupwarprunMode(pxConfig *config);
+static bool pendingcleanupwarpskyfileMode(pxConfig *config);
+static bool tocleanedwarpskyfileMode(pxConfig *config);
+
 static bool exportchipMode(pxConfig *config);
 static bool importchipMode(pxConfig *config);
 static bool exportwarpMode(pxConfig *config);
 static bool importwarpMode(pxConfig *config);
+
+static bool validDataState(psString data_state);
 
 // Tables to import/export
@@ -112,8 +120,11 @@
         MODECASE(BGTOOL_MODE_REVERTWARP,  revertwarpMode);
         MODECASE(BGTOOL_MODE_LISTWARP,    listwarpMode);
-        MODECASE(BGTOOL_MODE_TOCLEANCHIP, tocleanchipMode);
-        MODECASE(BGTOOL_MODE_CLEANEDCHIP, cleanedchipMode);
-        MODECASE(BGTOOL_MODE_TOCLEANWARP, tocleanwarpMode);
-        MODECASE(BGTOOL_MODE_CLEANEDWARP, cleanedwarpMode);
+        MODECASE(BGTOOL_MODE_PENDINGCLEANUPCHIPRUN, pendingcleanupchiprunMode);
+        MODECASE(BGTOOL_MODE_PENDINGCLEANUPCHIPIMFILE, pendingcleanupchipimfileMode);
+        MODECASE(BGTOOL_MODE_TOCLEANEDCHIPIMFILE, tocleanedchipimfileMode);
+        MODECASE(BGTOOL_MODE_UPDATECHIPIMFILE, updatechipimfileMode);
+        MODECASE(BGTOOL_MODE_PENDINGCLEANUPWARPRUN, pendingcleanupwarprunMode);
+        MODECASE(BGTOOL_MODE_PENDINGCLEANUPWARPSKYFILE, pendingcleanupwarpskyfileMode);
+        MODECASE(BGTOOL_MODE_TOCLEANEDWARPSKYFILE, tocleanedwarpskyfileMode);
         MODECASE(BGTOOL_MODE_EXPORTCHIP,  exportchipMode);
         MODECASE(BGTOOL_MODE_IMPORTCHIP,  importchipMode);
@@ -914,4 +925,5 @@
     return true;
 }
+
 static bool listchipMode(pxConfig *config)
 {
@@ -978,5 +990,5 @@
 }
 
-static bool tocleanchipMode(pxConfig *config)
+static bool pendingcleanupchiprunMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, NULL);
@@ -987,6 +999,7 @@
     psMetadata *where = psMetadataAlloc();
     pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "==");
-
-    psString query = pxDataGet("bgtool_tocleanchip.sql");
+    pxAddLabelSearchArgs(config, where, "-data_group", "chipBackgroundRun.data_group", "==");
+
+    psString query = pxDataGet("bgtool_pendingcleanupchiprun.sql");
     if (!query) {
         psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
@@ -1035,24 +1048,17 @@
     return true;
 }
-
-static bool cleanedchipMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
+static bool pendingcleanupchipimfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false);
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psMetadata *where = psMetadataAlloc();
+
     PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "==");
 
-    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
-    if (!pxIsValidCleanedState(state)) {
-        psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state);
-        return false;
-    }
-
-    if (!psListLength(where->list)) {
-        psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set.");
-        return false;
-    }
-
-    psString query = pxDataGet("bgtool_cleanedchip.sql");
+    psString query = pxDataGet("bgtool_pendingcleanupchipimfile.sql");
     if (!query) {
         psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
@@ -1060,5 +1066,5 @@
     }
 
-    {
+    if (where && psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
         psStringAppend(&query, " AND %s", whereClause);
@@ -1067,6 +1073,142 @@
     psFree(where);
 
-    if (!p_psDBRunQueryF(config->dbh, query, state)) {
-        psError(psErrorCodeLast(), false, "database error");
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) {
+        psError(psErrorCodeLast(), false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+
+    return true;
+}
+
+static bool change_imfile_data_state(pxConfig *config, psString data_state)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // chip_id, class_id are required
+    PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false);
+    PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
+
+    psString query = pxDataGet("bgtool_change_imfile_data_state.sql");
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_bg_id, class_id)) {
+        psFree(query);
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(query);
+    if (psDBAffectedRows(config->dbh) < 1) {
+        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
+        return false;
+    }
+
+    query = pxDataGet("bgtool_change_chiprun_state.sql");
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_bg_id, data_state)) {
+        psFree(query);
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(query);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool tocleanedchipimfileMode(pxConfig *config)
+{
+    return change_imfile_data_state(config, "cleaned");
+}
+
+static bool updatechipimfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false);
+    PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundImfile.chip_bg_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "==");
+
+    // optional
+
+    PXOPT_LOOKUP_STR(data_state, config->args, "-set_data_state", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false);
+    if (!data_state && !fault) {
+        psError(PS_ERR_UNKNOWN, true, "must supply either -set_fault or -set_data_state");
+        return false;
+    }
+    char *sep = "";
+    psString query = psStringCopy("UPDATE chipBackgroundImfile SET ");
+    if (data_state) {
+        if (!validDataState(data_state)) {
+            return false;
+        }
+        psStringAppend(&query, "%s data_state = '%s'", sep, data_state);
+        sep = ", ";
+    }
+    if (fault) {
+        psStringAppend(&query, "%s fault = %d", sep, fault);
+        sep = ", ";
+    }
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    } else {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+    psFree(where);
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
         return false;
     }
@@ -1449,5 +1591,5 @@
         return true;
     }
-    if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundSkyfile", !simple)) {
+    if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundSkyfile", !simple)) {
         psError(psErrorCodeLast(), false, "failed to print array");
         psFree(output);
@@ -1814,5 +1956,5 @@
 }
 
-static bool tocleanwarpMode(pxConfig *config)
+static bool pendingcleanupwarprunMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, NULL);
@@ -1824,5 +1966,5 @@
     pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "==");
 
-    psString query = pxDataGet("bgtool_tocleanwarp.sql");
+    psString query = pxDataGet("bgtool_pendingcleanupwarprun.sql");
     if (!query) {
         psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
@@ -1871,24 +2013,17 @@
     return true;
 }
-
-static bool cleanedwarpMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
+static bool pendingcleanupwarpskyfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false);
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psMetadata *where = psMetadataAlloc();
+
     PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "==");
 
-    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
-    if (!pxIsValidCleanedState(state)) {
-        psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state);
-        return false;
-    }
-
-    if (!psListLength(where->list)) {
-        psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set.");
-        return false;
-    }
-
-    psString query = pxDataGet("bgtool_cleanedwarp.sql");
+    psString query = pxDataGet("bgtool_pendingcleanupwarpskyfile.sql");
     if (!query) {
         psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
@@ -1896,5 +2031,5 @@
     }
 
-    {
+    if (where && psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
         psStringAppend(&query, " AND %s", whereClause);
@@ -1903,11 +2038,93 @@
     psFree(where);
 
-    if (!p_psDBRunQueryF(config->dbh, query, state)) {
-        psError(psErrorCodeLast(), false, "database error");
-        return false;
-    }
-    psFree(query);
-
-    return true;
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundSkyfile", !simple)) {
+        psError(psErrorCodeLast(), false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+
+    return true;
+}
+
+static bool change_skyfile_data_state(pxConfig *config, psString data_state)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // warp_id, skycell_id are required
+    PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false);
+    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
+
+    psString query = pxDataGet("bgtool_change_skyfile_data_state.sql");
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_bg_id, skycell_id)) {
+        psFree(query);
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(query);
+    if (psDBAffectedRows(config->dbh) < 1) {
+        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
+        return false;
+    }
+
+    query = pxDataGet("bgtool_change_warprun_state.sql");
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_bg_id, data_state)) {
+        psFree(query);
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(query);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool tocleanedwarpskyfileMode(pxConfig *config)
+{
+    return change_skyfile_data_state(config, "cleaned");
 }
 
@@ -1935,2 +2152,19 @@
     return importTables(config, infile, warpTables);
 }
+
+static bool validDataState(psString data_state)
+{
+    // NOTE: can't use pxIsValidState because
+    // update, scrubbed and purged are not supported for the backround stages (yet?)
+    if (!strcmp(data_state, "new") ||
+        !strcmp(data_state, "full") ||
+        !strcmp(data_state, "drop") ||
+        !strcmp(data_state, "cleaned") ||
+        !strcmp(data_state, "goto_cleaned") ||
+        !strcmp(data_state, "error_cleaned")) {
+        return true;
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "%s is not a valid value data_state", data_state);
+        return false;
+    }
+}