Changeset 32232

Timestamp:

Aug 30, 2011, 5:09:24 PM (15 years ago)

Author:

bills

Message:

use psFitsReadTableNew and psFitsWriteTableNew to reduce memory usage when censoring
sources by about 100 fold

File:

• trunk/magic/remove/src/streaksremove.c (modified) (7 diffs)

trunk/magic/remove/src/streaksremove.c

@@ -9 +9 @@
 
 #include "streaksremove.h"
+
+#include "psFitsTableNew.h"
 
 static pmConfig *parseArguments(int argc, char **argv);
@@ -302 +304 @@
     psLibFinalize();
 
-    fprintf(stderr, "Found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "streaksremove");
+    int leaks = psMemCheckLeaks (0, NULL, stdout, false);
+
+    fprintf(stderr, "Found %d leaks at %s\n", leaks, "streaksremove");
 
     return 0;
@@ -917 +921 @@
     sFile *out = sfiles->outSources;
 
-
     // Primary header, should be "something.hdr"
     {
@@ -960 +963 @@
         }
 
-        psArray *inTable = psFitsReadTable(in->fits);
-        if (!inTable) {
+#ifdef CHECK_MEM_USE
+        bool printStats = false;
+        size_t allocated = 0;
+        size_t persistent = 0;
+        size_t allocatedBefore = psMemStats(printStats, &allocated, &persistent);
+        printf("Before: %12lld %12lld %12lld\n", (long long) allocated, (long long) persistent, (long long) allocatedBefore);
+#endif
+
+        psFitsTable *table = psFitsReadTableNew(in->fits);
+
+#ifdef CHECK_MEM_USE
+        size_t allocatedAfter = psMemStats(printStats, &allocated, &persistent);
+        printf("After:  %12lld %12lld %12lld\n", (long long) allocated, (long long) persistent, (long long) allocatedAfter);
+        printf("Change: %12lld\n", (long long) allocatedAfter - allocatedBefore);
+#endif
+
+        if (!table) {
             psErrorStackPrint(stderr, "failed to read table in %s", in->resolved_name);
             streaksExit("", PS_EXIT_DATA_ERROR);
         }
 
-        psArray *outTable = psArrayAllocEmpty(inTable->n);
-        int j = 0;
+        // XXX: Handle case where numCols == 0 Can that happen?
+        // Previously we were writing a blank extension
+        int xCol = psFitsTableGetColumnIndex(table, "X_PSF");
+        assert(xCol >= 0 && xCol < table->numCols);
+
+        int yCol = psFitsTableGetColumnIndex(table, "Y_PSF");
+        assert(yCol >= 0 && yCol < table->numCols);
+
+        bool *rowCensored = psAlloc(table->numRows * sizeof(bool));
         int numCensored = 0;
-        for (int i = 0 ; i < inTable->n; i++) {
-            psMetadata *row = inTable->data[i];
-
-            psF32 x = psMetadataLookupF32(NULL, row, "X_PSF");
-            psF32 y = psMetadataLookupF32(NULL, row, "Y_PSF");
+
+        for (int i = 0 ; i < table->numRows; i++) {
+            psF32 x = psFitsTableGetCellF32(table, i, xCol);
+            psF32 y = psFitsTableGetCellF32(table, i, yCol);
 
             psImageMaskType mask;
@@ -984 +1008 @@
             // Key the source if the center pixel is not masked with maskStreak
             if (!(mask & maskStreak) ) {
-                psArraySet(outTable, j++, row);
+                // psArraySet(outTable, j++, row);
+                rowCensored[i] = false;
             } else {
+                rowCensored[i] = true;
                 numCensored++;
             }
         }
 
-        // get rid of unused elements (don't know if this is necessary)
-        psArrayRealloc(outTable, j);
+        if (!psFitsTableCensor(table, rowCensored)) {
+            streaksExit("", PS_EXIT_PROG_ERROR);
+        }
 
         addDestreakKeyword(header, true);
-        if (psArrayLength(outTable) > 0) {
+
+        if (table->numCols > 0) {
             printf("Censored %d sources\n", numCensored);
-            if (! psFitsWriteTable(out->fits, header, outTable, extname)) {
+            if (! psFitsWriteTableNew(out->fits, header, table, extname)) {
                 psErrorStackPrint(stderr, "failed to write table to %s", out->resolved_name);
                 streaksExit("", PS_EXIT_DATA_ERROR);
             }
-        } else if (psArrayLength(inTable) == 0) {
+        } else {
             printf("No input sources\n");
             // We'd like to write a blank table, but this is as good as it gets without more work to parse the
@@ -1008 +1036 @@
                 streaksExit("", PS_EXIT_DATA_ERROR);
             }
-        } else {
-            printf("Censored ALL %d sources\n", numCensored);
-            if (! psFitsWriteTableEmpty(out->fits, header, inTable->data[0], extname)) {
-                psErrorStackPrint(stderr, "failed to write empty table to %s", out->resolved_name);
-                streaksExit("", PS_EXIT_DATA_ERROR);
-            }
         }
         psFree(header);
-        psFree(outTable);
-        psFree(inTable);
+        psFree(table);
+        psFree(rowCensored);
     }
 
@@ -1049 +1071 @@
     }
 
-
     if (!psFitsClose(out->fits)) {
         psErrorStackPrint(stderr, "failed to close table %s", out->resolved_name);
         streaksExit("", PS_EXIT_DATA_ERROR);
     }
-}
+    out->fits = NULL;
+}