Changeset 42524 for branches/eam_branches/ipp-20230313/ippTools/src/fftool.c

Timestamp:

Aug 17, 2023, 4:35:50 PM (3 years ago)

Author:

eugene

Message:

merging from trunk: fftool defineforstacks mode added; options to ignore errors in pzgetext

Location:

branches/eam_branches/ipp-20230313/ippTools

Files:

• . (modified) (1 prop)
• src (modified) (1 prop)
• src/fftool.c (modified) (3 diffs)

branches/eam_branches/ipp-20230313/ippTools/src/fftool.c

@@ -33 +33 @@
 
 static bool definebyqueryMode(pxConfig *config);
+static bool defineforstacksMode(pxConfig *config);
 static bool updaterunMode(pxConfig *config);
 static bool todoMode(pxConfig *config);
@@ -68 +69 @@
     switch (config->mode) {
         MODECASE(FFTOOL_MODE_DEFINEBYQUERY,     definebyqueryMode);
+        MODECASE(FFTOOL_MODE_DEFINEFORSTACKS,   defineforstacksMode);
         MODECASE(FFTOOL_MODE_UPDATERUN,         updaterunMode);
         MODECASE(FFTOOL_MODE_TODO,              todoMode);
@@ -337 +339 @@
 }
 
+static bool defineforstacksMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    PXOPT_LOOKUP_STR(workdir,     config->args, "-set_workdir", true, false);
+    PXOPT_LOOKUP_STR(label,       config->args, "-set_label", true, false);
+
+    // optional
+    PXOPT_LOOKUP_STR(data_group,  config->args, "-set_data_group", false, false);
+    PXOPT_LOOKUP_STR(dist_group,  config->args, "-set_dist_group", false, false);
+    PXOPT_LOOKUP_STR(note,        config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_STR(reduction,   config->args, "-set_reduction", false, false);
+
+    PXOPT_LOOKUP_STR(sources_path_base,   config->args, "-set_sources_path_base", false, false);
+
+    psMetadata *skycalWhereMD = psMetadataAlloc();
+    pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_label",   "skycalRun.label",         "LIKE");
+    pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_data_group", "skycalRun.data_group", "LIKE");
+    PXOPT_COPY_S64(config->args, skycalWhereMD, "-select_skycal_id",      "skycalRun.skycal_id",     "==");
+
+    if (!psListLength(skycalWhereMD->list)) {
+        psError(PXTOOLS_ERR_CONFIG, false, "skycal search parameters are required");
+        psFree(skycalWhereMD);
+        return false;
+    }
+
+    PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_skycell_id",    "stackRun.skycell_id",      "LIKE");
+    PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_tess_id",       "stackRun.tess_id",         "==");
+    pxAddLabelSearchArgs(config, skycalWhereMD, "-select_filter",        "stackRun.filter",          "LIKE");
+    if (!pxskycellAddWhere(config, skycalWhereMD)) {
+        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
+        psFree(skycalWhereMD);
+        return false;
+    }
+
+    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    psString select = pxDataGet("fftool_defineforstacks.sql");
+    if (!select) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psFree(skycalWhereMD);
+        return false;
+    }
+
+    psString where = NULL;
+    psString whereClause = psDBGenerateWhereConditionSQL(skycalWhereMD, NULL);
+    psStringAppend(&where, "\nAND %s", whereClause);
+    psStringAppend(&select, " %s", where);
+    psFree(whereClause);
+    psFree(skycalWhereMD);
+
+    psString joinHook = NULL;
+    if (!rerun) {
+        psStringAppend(&joinHook, "\nLEFT JOIN fullForceRun ON fullForceRun.skycal_id = skycalRun.skycal_id");
+        psStringAppend(&joinHook, "\n %s\nAND fullForceRun.label = '%s'", where, label);
+        psStringAppend(&select, "\nAND ff_id IS NULL");
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, select, joinHook)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(select);
+        return false;
+    }
+    psFree(select);
+    psFree(joinHook);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+        psFree(where);
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psWarning("fftool: no rows found");
+        psFree(output);
+        psFree(where);
+        return true;
+    }
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "toFullForce", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            psFree(where);
+            return false;
+        }
+        psFree(output);
+        psFree(where);
+        return true;
+    }
+
+    psString warpQueryTemplate = pxDataGet("fftool_defineforstacks_select_warps.sql");
+
+    for (long i = 0; i < output->n; i++) {
+        psMetadata *row = output->data[i]; // Row from select
+        bool status;
+
+        if (!psDBTransaction(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(output);
+            return false;
+        }
+
+        psS64 skycal_id = psMetadataLookupS64(&status, row, "skycal_id");
+
+        psString path_base = NULL;
+        if (sources_path_base) {
+            path_base = sources_path_base;
+        } else {
+            path_base = psMetadataLookupStr(&status, row, "path_base");
+            psAssert(status, "failed to find skycal path_base?");
+        }
+
+        psString skycal_data_group = psMetadataLookupStr(&status, row, "data_group");
+
+        psString query = NULL;
+        psStringAppend(&query, warpQueryTemplate, skycal_id);
+
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(query);
+            return false;
+        }
+        psFree(query);
+
+        // Find the warps for this skycell and filter combination
+        psArray *warpOutput = p_psDBFetchResult(config->dbh);
+        if (!warpOutput) {
+            psErrorCode err = psErrorCodeLast();
+            switch (err) {
+                case PS_ERR_DB_CLIENT:
+                    psError(PXTOOLS_ERR_SYS, false, "database error");
+                case PS_ERR_DB_SERVER:
+                    psError(PXTOOLS_ERR_PROG, false, "database error");
+                default:
+                    psError(PXTOOLS_ERR_PROG, false, "unknown error");
+            }
+            return false;
+        }
+        if (!psArrayLength(warpOutput)) {
+            // no warps for this skycal. Suprise?
+            psFree(warpOutput);
+            psFree(query);
+            continue;
+        }
+
+        // create a staticskyRun
+        if (!fullForceRunInsert(config->dbh,
+                                0x0,         // ff_id
+                                skycal_id,
+                                path_base,
+                                "new",       // state
+                                workdir,
+                                label,
+                                data_group ? data_group : (skycal_data_group ? skycal_data_group : label),
+                                dist_group,
+                                note,
+                                reduction,
+                                NULL        // registered
+                )
+            ) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction");
+            }
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(output);
+            return false;
+        }
+
+        psS64 ff_id = psDBLastInsertID(config->dbh);
+
+        for (int j = 0; j < warpOutput->n; j++) {
+            psMetadata *warpRow = warpOutput->data[j];
+            psS64 warp_id = psMetadataLookupS64(&status, warpRow, "warp_id");
+
+            if (!fullForceInputInsert(config->dbh,
+                                ff_id,
+                                warp_id)
+               ) {
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction");
+                }
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(warpOutput);
+                psFree(output);
+                return false;
+            }
+        }
+
+        if (!psDBCommit(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(warpOutput);
+            psFree(output);
+            return false;
+        }
+        psFree(warpOutput);
+    }
+    psFree(output);
+
+    return true;
+}
+
 static bool updaterunMode(pxConfig *config)
 {