IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 30, 2012, 1:46:12 PM (14 years ago)
Author:
eugene
Message:

merge changes from eam_branches/ipp-20120405

Location:
trunk
Files:
28 edited
2 copied

Legend:

Unmodified
Added
Removed

  • trunk

  • trunk/Ohana/src/opihi/cmd.astro

  • trunk/Ohana/src/opihi/cmd.astro/Makefile

    r33662 r33963  
    4646$(SRC)/ringflux.$(ARCH).o          \
    4747$(SRC)/petrosian.$(ARCH).o         \
     48$(SRC)/kronflux.$(ARCH).o          \
    4849$(SRC)/multifit.$(ARCH).o          \
    4950$(SRC)/objload.$(ARCH).o           \

  • trunk/Ohana/src/opihi/cmd.astro/init.c

    r33662 r33963  
    3232int ringflux                PROTO((int, char **));
    3333int petrosian               PROTO((int, char **));
     34int kronflux                PROTO((int, char **));
    3435int mkgauss                 PROTO((int, char **));
    3536int mksersic                PROTO((int, char **));
    … …  
    8889  {1, "ringflux",    ringflux,     "mean flux in a ring"},
    8990  {1, "petrosian",   petrosian,    "petrosian parameters given radial bins"},
     91  {1, "kronflux",    kronflux,     "measure kronflux stats"},
    9092  {1, "multifit",    multifit,     "fit multi-order spectrum"},
    9193  {1, "objload",     objload,      "plot obj data on Ximage "},

  • trunk/Ohana/src/opihi/cmd.basic/Makefile

    r31635 r33963  
    4444$(SRC)/quit.$(ARCH).o        \
    4545$(SRC)/run_for.$(ARCH).o    \
     46$(SRC)/run_foreach.$(ARCH).o    \
    4647$(SRC)/run_if.$(ARCH).o     \
    4748$(SRC)/run_while.$(ARCH).o  \

  • trunk/Ohana/src/opihi/cmd.basic/init.c

    r27255 r33963  
    3131int quit            PROTO((int, char **));
    3232int run_for         PROTO((int, char **));
     33int run_foreach     PROTO((int, char **));
    3334int run_if          PROTO((int, char **));
    3435int run_while       PROTO((int, char **));
    … …  
    8182  {1, "quit",          quit,               "exit program *"},
    8283  {1, "for",           run_for,            "for loop"},
     84  {1, "foreach",       run_foreach,        "foreach loop"},
    8385  {1, "if",            run_if,             "logical cases *"},
    8486  {1, "while",         run_while,          "while loop"},

  • trunk/Ohana/src/opihi/cmd.basic/input.c

    r7917 r33963  
    4040  fclose (infile);
    4141
     42  if (!inlist.Nlines) gprint (GP_ERR, "WARNING: input file (%s) was empty\n", argv[1]);
     43
    4244  /* process this list */
    4345  status = exec_loop (&inlist);

  • trunk/Ohana/src/opihi/cmd.basic/run_for.c

    r31667 r33963  
    55int run_for (int argc, char **argv) {
    66
    7   int ThisList, depth, i, done, status, found, NLINES, j;
     7  int N, ThisList, depth, i, done, status, found, NLINES, j;
    88  double start, end, delta;
    99  char *input, *endptr;
    1010  double value, sign;
    1111  Macro loop;
     12
     13  int Inclusive = FALSE;
     14  if ((N = get_argument (argc, argv, "-incl"))) {
     15    remove_argument (N, &argc, argv);
     16    Inclusive = TRUE;
     17  }
    1218
    1319  if ((argc != 4) && (argc != 5)) {
    … …  
    95101  REALLOCATE (loop.line, char *, MAX (loop.Nlines, 1));
    96102
     103  if (Inclusive) end += 0.01*delta;
     104
    97105  status = TRUE;
    98106  interrupt = FALSE;

  • trunk/Ohana/src/opihi/cmd.data/dot.c

    r14181 r33963  
    33int dot (int argc, char **argv) {
    44 
    5   int kapa;
     5  int kapa, N;
    66  Graphdata graphmode;
    77  float x, y;
    88
    99  if (!style_args (&graphmode, &argc, argv, &kapa)) return FALSE;
     10
     11  /* FracPositions uses coordinates of 0-1 relative to axis range */
     12  int FracPositions = FALSE;
     13  if ((N = get_argument (argc, argv, "-frac"))) {
     14    remove_argument (N, &argc, argv);
     15    FracPositions = TRUE;
     16  }
    1017
    1118  if (argc != 3) {
    … …  
    1522  x = atof(argv[1]);
    1623  y = atof(argv[2]);
     24
     25  if (FracPositions) {
     26    x =  x * (graphmode.xmax - graphmode.xmin) + graphmode.xmin;
     27    y =  y * (graphmode.ymax - graphmode.ymin) + graphmode.ymin;
     28  }   
    1729
    1830  /* set point style and errorbar mode (these are NOT sticky) */

  • trunk/Ohana/src/opihi/cmd.data/gridify.c

    r31450 r33963  
    55  int i, Nx, Ny, Xb, Yb, Normalize, N;
    66  float Xmin, Xmax, dX, Ymin, Ymax, dY, initValue;
    7   float *buf, *val;
     7  float *buf, *val, *cnt;
    88  int *Nval;
    9   Buffer *bf;
    109  Vector *vx, *vy, *vz;
    1110  opihi_flt *x, *y, *z;
     11
     12  Buffer *bf = NULL;
     13  Buffer *ct = NULL;
    1214
    1315  Normalize = TRUE;
    … …  
    1517    remove_argument (N, &argc, argv);
    1618    Normalize = FALSE;
     19    if ((ct = SelectBuffer (argv[N], ANYBUFFER, TRUE)) == NULL) return (FALSE);
     20    remove_argument (N, &argc, argv);
    1721  }
    1822
    … …  
    2428  }
    2529
    26   if (argc != 11) {
    27     gprint (GP_ERR, "USAGE: gridify x y z buffer Xmin Xmax dX Ymin Ymax dY\n");
     30  Xmin = Xmax = dX = NAN;
     31  if ((N = get_argument (argc, argv, "-x"))) {
     32    remove_argument (N, &argc, argv);
     33    Xmin = atof (argv[N]);
     34    remove_argument (N, &argc, argv);
     35    Xmax = atof (argv[N]);
     36    remove_argument (N, &argc, argv);
     37    dX   = atof (argv[N]);
     38    remove_argument (N, &argc, argv);
     39  }   
     40
     41  Ymin = Ymax = dY = NAN;
     42  if ((N = get_argument (argc, argv, "-y"))) {
     43    remove_argument (N, &argc, argv);
     44    Ymin = atof (argv[N]);
     45    remove_argument (N, &argc, argv);
     46    Ymax = atof (argv[N]);
     47    remove_argument (N, &argc, argv);
     48    dY   = atof (argv[N]);
     49    remove_argument (N, &argc, argv);
     50  }   
     51
     52  if (argc != 5) {
     53    gprint (GP_ERR, "USAGE: gridify x y z buffer [-x Xmin Xmax dX] [-y Ymin Ymax dY] [-init-value value] [-raw]\n");
    2854    return (FALSE);
    2955  }
    … …  
    4167  REQUIRE_VECTOR_FLT (vz, FALSE);
    4268
    43   Xmin = atof (argv[5]);
    44   Xmax = atof (argv[6]);
    45   dX   = atof (argv[7]);
     69  if (isnan(dX)) {
     70    Xmin = 0;
     71    Xmax = bf[0].matrix.Naxis[0];
     72    dX = 1;
     73  }
    4674
    47   Ymin = atof (argv[8]);
    48   Ymax = atof (argv[9]);
    49   dY   = atof (argv[10]);
     75  if (isnan(dY)) {
     76    Ymin = 0;
     77    Ymax = bf[0].matrix.Naxis[1];
     78    dY = 1;
     79  }
    5080
    51   Nx = (Xmax - Xmin) / dX + 1;
    52   Ny = (Ymax - Ymin) / dY + 1;
     81  Nx = (Xmax - Xmin) / dX;
     82  Ny = (Ymax - Ymin) / dY;
    5383 
    54   gfits_free_matrix (&bf[0].matrix);
    55   gfits_free_header (&bf[0].header);
    56   CreateBuffer (bf, Nx, Ny, -32, 0.0, 1.0);
    57   strcpy (bf[0].file, "(empty)");
     84  if ((Nx != bf[0].matrix.Naxis[0]) || (Ny != bf[0].matrix.Naxis[1])) {
     85    gfits_free_matrix (&bf[0].matrix);
     86    gfits_free_header (&bf[0].header);
     87    CreateBuffer (bf, Nx, Ny, -32, 0.0, 1.0);
     88    strcpy (bf[0].file, "(empty)");
     89  }
    5890
    5991  ALLOCATE (val, float, Nx*Ny);
    … …  
    68100    Xb = (*x - Xmin) / dX;
    69101    Yb = (*y - Ymin) / dY;
     102    if (Xb < 0) continue;
     103    if (Yb < 0) continue;
    70104    if (Xb >= Nx) continue;
    71105    if (Yb >= Ny) continue;
    … …  
    74108  }
    75109
     110  if (!Normalize) {
     111    gfits_free_matrix (&ct[0].matrix);
     112    gfits_free_header (&ct[0].header);
     113    CreateBuffer (ct, Nx, Ny, -32, 0.0, 1.0);
     114    strcpy (ct[0].file, "(empty)");
     115
     116    buf = (float *) bf[0].matrix.buffer;
     117    cnt = (float *) ct[0].matrix.buffer;
     118    for (i = 0; i < Nx*Ny; i++) {
     119      if (Nval[i] == 0) continue;
     120      buf[i] = val[i];
     121      cnt[i] = Nval[i];
     122    }
     123    free (val);
     124    free (Nval);
     125    return TRUE;
     126  }
     127
    76128  buf = (float *) bf[0].matrix.buffer;
    77129  for (i = 0; i < Nx*Ny; i++) {
    78130    buf[i] = initValue;
    79     if (Normalize) {
    80       if (Nval[i] == 0) {
    81         continue;
    82       }
    83       buf[i] = val[i] / Nval[i];
    84     } else {
    85       buf[i] = val[i];
    86     }
     131    if (Nval[i] == 0) continue;
     132    buf[i] = val[i] / Nval[i];
    87133  }
    88134

  • trunk/Ohana/src/opihi/cmd.data/match2d.c

    r30610 r33963  
    167167  NMATCH = X1->Nelements;
    168168  ResetVector (index, OPIHI_INT, NMATCH);
     169
     170  for (i = 0; i < index->Nelements; i++) { index->elements.Int[i] = -1; }
    169171
    170172  ALLOCATE (N1, off_t, X1->Nelements);

  • trunk/Ohana/src/opihi/cmd.data/threshold.c

    r33662 r33963  
    6969    x0 = isFltX ? vecx[0].elements.Flt[Nhi] : vecy[0].elements.Int[Nhi];
    7070    x1 = isFltX ? vecx[0].elements.Flt[Nhi+1]   : vecy[0].elements.Int[Nhi+1];
    71     Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
     71    if (y0 == y1) {
     72      Xvalue = 0.5*(x0 + x1);
     73    } else {
     74      Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
     75    }   
    7276  } else {
    7377    // interpolate to value:
    … …  
    7680    x0 = isFltX ? vecx[0].elements.Flt[Nhi-1] : vecy[0].elements.Int[Nhi-1];
    7781    x1 = isFltX ? vecx[0].elements.Flt[Nhi]   : vecy[0].elements.Int[Nhi];
    78     Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
     82    if (y0 == y1) {
     83      Xvalue = 0.5*(x0 + x1);
     84    } else {
     85      Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
     86    }
    7987  }
    8088

  • trunk/Ohana/src/opihi/cmd.data/vgroup.c

    r25757 r33963  
    11# include "data.h"
     2
     3enum {USE_MEDIAN, USE_COUNT, USE_SUM, USE_MEAN};
    24
    35int vgroup (int argc, char **argv) {
    … …  
    2325  // }
    2426
     27  int mode = USE_MEDIAN;
     28  if ((N = get_argument (argc, argv, "-sum"))) {
     29    remove_argument (N, &argc, argv);
     30    mode = USE_SUM;
     31  }
     32  if ((N = get_argument (argc, argv, "-mean"))) {
     33    remove_argument (N, &argc, argv);
     34    mode = USE_MEAN;
     35  }
     36
     37  float binsize = NAN;
     38  if ((N = get_argument (argc, argv, "-binsize"))) {
     39    remove_argument (N, &argc, argv);
     40    binsize = atof(argv[N]);
     41    remove_argument (N, &argc, argv);
     42  }
     43
    2544  if (argc != 5) {
    26     gprint (GP_ERR, "USAGE: vbin <xin> <yin> <xout> <yout>\n");
     45    gprint (GP_ERR, "USAGE: vgroup <xin> <yin> <xout> <yout>\n");
    2746    gprint (GP_ERR, " group x,y values in bins defined by <xout>\n");
     47    gprint (GP_ERR, " by default, yout has the median of the associated input values\n");
     48    gprint (GP_ERR, " use -sum to add values in the bin\n");
     49    gprint (GP_ERR, " use <yin> = histogram count matching values\n");
     50    gprint (GP_ERR, " use -binsize to specify a fixed bin width (<xout> will define the bin center)\n");
    2851    return (FALSE);
    2952  }
    3053
    3154  if ((xin  = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE);
    32   if ((yin  = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE);
    3355  if ((xout = SelectVector (argv[3], OLDVECTOR, TRUE)) == NULL) return (FALSE);
    3456  if ((yout = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE);
     57  yin = NULL;
     58
     59  // this should conflict with the -sum option...
     60  if (!strcmp(argv[2], "histogram")) {
     61    mode = USE_COUNT;
     62  } else {
     63    if ((yin  = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE);
     64  }
    3565
    3666  // re-binning creates a float vector
    … …  
    4171
    4272  for (i = 0; i < xout[0].Nelements - 1; i++) {
    43     xmin = xout[0].elements.Flt[i];
    44     xmax = xout[0].elements.Flt[i+1];
     73    if (isnan(binsize)) {
     74      xmin = xout[0].elements.Flt[i];
     75      xmax = xout[0].elements.Flt[i+1];
     76    } else {
     77      xmin = xout[0].elements.Flt[i] - 0.5*binsize;
     78      xmax = xout[0].elements.Flt[i] + 0.5*binsize;
     79    }
    4580
    4681    N = 0;
    … …  
    4883      if (xin[0].elements.Flt[j] < xmin) continue;
    4984      if (xin[0].elements.Flt[j] > xmax) continue;
    50       values[N] = yin[0].elements.Flt[j];
     85      if (yin) {
     86        values[N] = yin[0].elements.Flt[j];
     87      }
    5188      N++;
    5289    }
    5390   
    54     dsort (values, N);
    55     if (N > 1) {
    56       sum = (N % 2) ? values[(int)(0.5*N)] : 0.5*(values[N/2] + values[N/2 + 1]);
    57     } else {
    58       sum = values[0];
     91    sum = NAN;
     92    switch (mode) {
     93      case USE_MEDIAN:
     94        dsort (values, N);
     95        if (N > 1) {
     96          sum = (N % 2) ? values[(int)(0.5*N)] : 0.5*(values[N/2] + values[N/2 + 1]);
     97        } else {
     98          sum = values[0];
     99        }
     100        break;
     101
     102      case USE_SUM:
     103        sum = 0.0;
     104        for (j = 0; j < N; j++) {
     105          sum += values[j];
     106        }
     107        break;
     108       
     109      case USE_COUNT:
     110        sum = N;
     111        break;
     112
     113      case USE_MEAN:
     114        sum = 0.0;
     115        for (j = 0; j < N; j++) {
     116          sum += values[j];
     117        }
     118        sum /= N;
     119        break;
    59120    }
    60 
    61     // measure the stat for this bin
    62     // sum = 0.0;
    63     // for (j = 0; j < N; j++) {
    64     //   sum += values[j];
    65     // }
    66     // sum /= N;
    67121    yout[0].elements.Flt[i] = sum;
    68122  }

  • trunk/Ohana/src/opihi/cmd.data/vstats.c

    r32632 r33963  
    144144      }     
    145145    }
    146     stdev = sqrt (var / N);
     146    stdev = sqrt (var / (N - 1));
    147147
    148148    Nmode = 0;

  • trunk/Ohana/src/opihi/cmd.data/write_vectors.c

    r33662 r33963  
    2121  FITS = NULL;
    2222  if ((N = get_argument (argc, argv, "-fits"))) {
     23    if (format) {
     24      gprint (GP_ERR, "ERROR: do not mix -fits and -format\n");
     25      free (format);
     26      return (FALSE);
     27    }
    2328    remove_argument (N, &argc, argv);
    2429    FITS = strcreate (argv[N]);
    2530    remove_argument (N, &argc, argv);
     31  }
     32
     33  /* option generate a FITS output table */
     34  int CSV = FALSE;
     35  if ((N = get_argument (argc, argv, "-csv"))) {
     36    if (format) {
     37      gprint (GP_ERR, "ERROR: do not mix -csv and -format\n");
     38      free (format);
     39      return (FALSE);
     40    }
     41    if (FITS) {
     42      gprint (GP_ERR, "ERROR: do not mix -csv and -fits\n");
     43      free (FITS);
     44      return (FALSE);
     45    }
     46    remove_argument (N, &argc, argv);
     47    CSV = TRUE;
    2648  }
    2749
    … …  
    3052    remove_argument (N, &argc, argv);
    3153    append = TRUE;
     54  }
     55
     56  int ADD_HEADER = FALSE;
     57  if ((N = get_argument (argc, argv, "-header"))) {
     58    remove_argument (N, &argc, argv);
     59    ADD_HEADER = TRUE;
    3260  }
    3361
    … …  
    83111
    84112  /* default output format */
     113  if (ADD_HEADER) {
     114    for (j = 0; j < Nvec; j++) {
     115      if (CSV) {
     116        fprintf (f, "%s,", vec[j][0].name);
     117      } else {
     118        if (j == 0) fprintf (f, "# ");
     119        fprintf (f, "%s ", vec[j][0].name);
     120      }
     121    }
     122    fprintf (f, "\n");
     123  }
     124
     125  /* default output format */
    85126  if (format == (char *) NULL) {
    86127    for (i = 0; i < vec[0][0].Nelements; i++) {
    87128      for (j = 0; j < Nvec; j++) {
    88129        if (vec[j][0].type == OPIHI_FLT) {
    89           fprintf (f, "%.12g ", vec[j][0].elements.Flt[i]);
     130          if (CSV) {
     131            fprintf (f, "%.12g,", vec[j][0].elements.Flt[i]);
     132          } else {
     133            fprintf (f, "%.12g ", vec[j][0].elements.Flt[i]);
     134          }
    90135        } else {
    91           fprintf (f, "%d ", vec[j][0].elements.Int[i]);
     136          if (CSV) {
     137            fprintf (f, "%d,", vec[j][0].elements.Int[i]);
     138          } else {
     139            fprintf (f, "%d ", vec[j][0].elements.Int[i]);
     140          }
    92141        }
    93142      }

  • trunk/Ohana/src/opihi/dvo/avextract.c

    r33662 r33963  
    3737  }
    3838
     39  int VERBOSE2 = FALSE;
     40  if ((N = get_argument (argc, argv, "-vv"))) {
     41    remove_argument (N, &argc, argv);
     42    VERBOSE = TRUE;
     43    VERBOSE2 = TRUE;
     44  }
     45
    3946  int PARALLEL = FALSE;
    4047  if ((N = get_argument (argc, argv, "-parallel"))) {
    … …  
    6572      return FALSE;
    6673    }   
     74  }
     75
     76  // this is used to NOT save the results in the results file
     77  // use this option when mextract is used in a script which does its
     78  // own job of packaging the results
     79  int SKIP_RESULTS = FALSE;
     80  if ((N = get_argument (argc, argv, "-skip-results"))) {
     81    remove_argument (N, &argc, argv);
     82    SKIP_RESULTS = TRUE;
    6783  }
    6884
    … …  
    8096    goto escape;
    8197  }
     98
     99  // init locally static variables (time refs)
     100  dbExtractAveragesInit ();
    82101
    83102  // command-line is of the form: avextract field,field, field [where (field op value)...]
    … …  
    173192    catalog.filename = (HOST_ID || PARALLEL_LOCAL) ? hostfile : skylist[0].filename[i];
    174193    catalog.catflags = LOAD_AVES | LOAD_SECF;
    175     if (needMeasures) {
    176       catalog.catflags |= LOAD_MEAS;
    177     }
     194    catalog.catflags |= needMeasures ? LOAD_MEAS : SKIP_MEAS;
    178195    catalog.Nsecfilt = 0;
    179196
    … …  
    181198     
    182199    // an error exit status here is a significant error
    183     if (!dvo_catalog_open (&catalog, NULL, FALSE, "r")) {
     200    if (!dvo_catalog_open (&catalog, NULL, VERBOSE2, "r")) {
    184201      gprint (GP_ERR, "ERROR: failure to open catalog file %s\n", catalog.filename);
    185202      exit (2);
    … …  
    193210      m = catalog.average[j].measureOffset;
    194211
    195       // reset counters for saved fields, extract fields
    196       dbExtractAveragesInit ();
     212      dbExtractAveragesInitAve ();  // reset counters for saved fields (costs very little)
     213
    197214      for (n = 0; n < Nfields; n++) {
    198215        values[n] = dbExtractAverages (&catalog.average[j], &catalog.secfilt[j*Nsecfilt], &catalog.measure[m], &fields[n]);
    … …  
    229246
    230247  // write vectors to a table (this is used by parallel dvo operations, but can be used elsewhere)
    231   if (RESULT_FILE) {
     248  if (RESULT_FILE && !SKIP_RESULTS) {
    232249    int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, Nreturn, FALSE, NULL);
    233     if (!status) goto escape;
     250    if (!status) {
     251      goto escape;
     252    }
    234253  }
    235254

  • trunk/Ohana/src/opihi/dvo/avmatch.c

    r33662 r33963  
    6666  if (!InitPhotcodes ()) goto escape;
    6767  Nsecfilt = GetPhotcodeNsecfilt ();
     68
     69  // init locally static variables (time refs)
     70  dbExtractAveragesInit ();
    6871
    6972  // this does all the work of re-packaging the command, calling it on the remote machines, then loading in the results
    … …  
    129132  }
    130133
     134  // check the requested fields : are all average/secfilt entries, or do we need measures?
     135  int needMeasures = FALSE;
     136  for (i = 0; !needMeasures && (i < Nfields); i++) {
     137    if (fields[i].magMode == MAG_NONE) continue;
     138    if (fields[i].photcode == NULL) continue; // assert this?
     139    if (fields[i].photcode[0].type == PHOT_REF) needMeasures = TRUE;
     140    if (fields[i].photcode[0].type == PHOT_DEP) needMeasures = TRUE;
     141  }
     142
    131143  /* load regions which contain all supplied RA,DEC coordinates */
    132144  if ((skylist = SelectRegionsByCoordVectors (RAvec, DECvec)) == NULL) goto escape;
    … …  
    168180    snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
    169181    catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i];
    170     catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF;
     182    catalog.catflags = LOAD_AVES | LOAD_SECF;
     183    catalog.catflags |= needMeasures ? LOAD_MEAS : SKIP_MEAS;
    171184    catalog.Nsecfilt = 0;
    172185
    … …  
    196209
    197210      // reset counters for saved fields, extract fields
    198       dbExtractAveragesInit ();
     211      dbExtractAveragesInitAve ();
    199212      for (n = 0; n < Nfields; n++) {
    200213        values[n] = dbExtractAverages (&catalog.average[Ncat], &catalog.secfilt[Ncat*Nsecfilt], &catalog.measure[m], &fields[n]);

  • trunk/Ohana/src/opihi/dvo/dvo_client.c

    r33662 r33963  
    9898
    9999  int status = input (2, &argv[0]);
    100   if (!status) exit (2);
     100  if (!status) {
     101    fprintf (stderr, "WARNING: exit status 2 %d (%s)\n", HOST_ID, HOSTDIR);
     102    exit (2);
     103  }
    101104  exit (0);
    102105

  • trunk/Ohana/src/opihi/dvo/dvo_host_utils.c

    r33662 r33963  
    44# define PARALLEL_MANUAL 0
    55# define PARALLEL_SERIAL 0
    6 # define MAX_PATH_LENGTH 1024
    7 
    8 int HostTableLaunchJobs (HostTable *table, char *basecmd, char *options) {
     6# define DVO_MAX_PATH 1024
     7
     8# define DIE(WHO,MSG) { perror(WHO); myAbort(MSG); }
     9
     10int HostTableLaunchJobs (HostTable *table, char *basecmd, char *options, int VERBOSE) {
    911
    1012  char uniquer[12];
    … …  
    1315  snprintf (uniquer, 12, "%05d.%05d", PID, TIME % 100000);
    1416
     17  int top_status = TRUE;
    1518  int i;
    1619  for (i = 0; i < table->Nhosts; i++) {
    1720
    1821    // ensure that the paths are absolute path names
    19     char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH);
     22    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
    2023    free (table->hosts[i].pathname);
    2124    table->hosts[i].pathname = tmppath;
    … …  
    2326    // need to save the results filename with the uniquer
    2427    // XXX a bit of a waste (but only 1024 * 60 bytes or so
    25     ALLOCATE (table->hosts[i].results, char, MAX_PATH_LENGTH);
    26     snprintf (table->hosts[i].results, MAX_PATH_LENGTH, "%s/dvo.results.%s.fits", table->hosts[i].pathname, uniquer);
    27 
    28     char commandBase[MAX_PATH_LENGTH];
    29     snprintf (commandBase, MAX_PATH_LENGTH, "dvo.command.%s.txt", uniquer);
    30     char *commandFile = abspath(commandBase, MAX_PATH_LENGTH);
     28    ALLOCATE (table->hosts[i].results, char, DVO_MAX_PATH);
     29    snprintf (table->hosts[i].results, DVO_MAX_PATH, "%s/dvo.results.%s.fits", table->hosts[i].pathname, uniquer);
     30
     31    char commandBase[DVO_MAX_PATH];
     32    snprintf (commandBase, DVO_MAX_PATH, "dvo.command.%s.txt", uniquer);
     33    char *commandFile = abspath(commandBase, DVO_MAX_PATH);
    3134
    3235    FILE *f = fopen (commandFile, "w");
    3336    fprintf (f, "%s\n", basecmd);
    34     fclose (f);
     37    if (fflush (f)) DIE("flush", "failed to flush");
     38
     39    int fd = fileno (f);
     40    if (fsync (fd)) DIE("fsync", "failed to fsync");
     41
     42    if (fclose (f)) DIE("close", "failed to close");
     43
     44    // force NFS to write the file to disk
     45    int state;
     46    f = fsetlockfile (commandFile, 0.5, LCK_XCLD, &state);
     47    fclearlockfile (commandFile, f, LCK_XCLD, &state);
    3548
    3649    char command[1024];
    … …  
    3851    free (commandFile);
    3952
    40     if (DEBUG || PARALLEL_MANUAL) fprintf (stderr, "command: %s\n", command);
     53    if (VERBOSE) gprint (GP_ERR, "command: %s\n", command);
    4154
    4255    if (PARALLEL_MANUAL) {
    … …  
    4760      int status = system (command);
    4861      if (status) {
    49         fprintf (stderr, "ERROR running relphot_client\n");
    50         exit (2);
     62        gprint (GP_ERR, "ERROR running relphot_client\n");
     63        top_status = FALSE;
    5164      }
    5265    } else {
    … …  
    5568      int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
    5669      if (!pid) {
    57         if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
    58         exit (1);
     70        gprint (GP_ERR, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
     71        top_status = FALSE;
     72        continue;
    5973      }
    6074      table->hosts[i].pid = pid; // save for future reference
    6175    }
    6276  }
    63   return TRUE;
     77  return top_status;
    6478}
    6579
    … …  
    86100  }
    87101
    88   char *CATDIR = abspath (tmppath, MAX_PATH_LENGTH);
     102  char *CATDIR = abspath (tmppath, DVO_MAX_PATH);
    89103  if (!CATDIR) {
    90104    gprint (GP_ERR, "failed to make an absolute path from %s (too long)\n", tmppath);
    … …  
    126140
    127141  // launch this command remotely
    128   HostTableLaunchJobs (table, basecmd, options);
     142  HostTableLaunchJobs (table, basecmd, options, VERBOSE);
    129143  free (options);
    130144  free (basecmd);
    … …  
    184198  return TRUE;
    185199}
     200
     201// re-gather the remote results files: this can be used in case one of the clients failed,
     202// and has since been re-run
     203int HostTableReloadResults (char *uniquer, int VERBOSE) {
     204
     205  int i;
     206
     207  // load the list of hosts
     208  SkyTable *sky = GetSkyTable();
     209  if (!sky) {
     210    gprint (GP_ERR, "failed to load sky table for database\n");
     211    return FALSE;
     212  }
     213
     214  char *CATDIR = GetCATDIR ();
     215  if (!CATDIR) {
     216    gprint (GP_ERR, "failed to get CATDIR for database\n");
     217    return FALSE;
     218  }
     219
     220  HostTable *table = HostTableLoad (CATDIR, sky->hosts);
     221  if (!table) {
     222    gprint (GP_ERR, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
     223    return FALSE;
     224  }   
     225
     226  // load fields from file
     227  int    Nvec = 0;
     228  Vector **vec = NULL;
     229  for (i = 0; i < table->Nhosts; i++) {
     230    // ensure that the paths are absolute path names
     231    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
     232    free (table->hosts[i].pathname);
     233    table->hosts[i].pathname = tmppath;
     234
     235    // need to save the results filename with the uniquer
     236    // XXX a bit of a waste (but only 1024 * 60 bytes or so
     237    ALLOCATE (table->hosts[i].results, char, DVO_MAX_PATH);
     238    snprintf (table->hosts[i].results, DVO_MAX_PATH, "%s/dvo.results.%s.fits", table->hosts[i].pathname, uniquer);
     239
     240    int    Ninvec = 0;
     241    Vector **invec = ReadVectorTableFITS (table->hosts[i].results, "RESULT", &Ninvec);
     242    if (!invec) {
     243      // failed to read the file, now what?
     244      gprint (GP_ERR, "failed to read remote result file : %s\n", table->hosts[i].results);
     245      free (table->hosts[i].results);
     246      table->hosts[i].results = NULL;
     247      continue;
     248    }
     249    free (table->hosts[i].results);
     250    table->hosts[i].results = NULL;
     251
     252    // fprintf (stderr, "%s : %d\n", table->hosts[i].pathname, invec[0]->Nelements);
     253
     254    vec = MergeVectors (vec, &Nvec, invec, Ninvec);
     255    if (vec != invec) {
     256      FreeVectorArray (invec, Ninvec);
     257    }
     258  }
     259
     260  for (i = 0; i < Nvec; i++) {
     261    AssignVector (vec[i], vec[i]->name, ANYVECTOR, TRUE);
     262  }
     263  free (vec);
     264
     265  free (table);
     266  return TRUE;
     267}

  • trunk/Ohana/src/opihi/dvo/gstar.c

    r33662 r33963  
    116116
    117117  /* lock, load, unlock catalog */
    118   catalog.catflags = GetMeasures ? LOAD_AVES | LOAD_MEAS | LOAD_SECF : LOAD_AVES | LOAD_SECF;
     118  catalog.catflags = LOAD_AVES | LOAD_SECF;
     119  catalog.catflags |= GetMeasures ? LOAD_MEAS : SKIP_MEAS;
    119120  catalog.Nsecfilt = 0;
    120121

  • trunk/Ohana/src/opihi/dvo/hosts.c

    r33662 r33963  
    11# include "dvoshell.h"
    22# include <glob.h>
    3 # define MAX_PATH_LENGTH 1024
     3# define DVO_MAX_PATH 1024
    44
    55// functions to manage the remote hosts
    … …  
    5959    for (i = 0; i < table->Nhosts; i++) {
    6060      pglob.gl_offs = 0;
    61       char name[MAX_PATH_LENGTH];
    62       snprintf (name, MAX_PATH_LENGTH, "%s/dvo.results.%05d.*.fits", table->hosts[i].pathname, PID);
     61      char name[DVO_MAX_PATH];
     62      snprintf (name, DVO_MAX_PATH, "%s/dvo.results.%05d.*.fits", table->hosts[i].pathname, PID);
    6363      if (VERBOSE) gprint (GP_ERR, "checking %s\n", name);
    6464      glob (name, 0, NULL, &pglob);

  • trunk/Ohana/src/opihi/dvo/mextract.c

    r33662 r33963  
    189189      }
    190190
    191       dbExtractMeasuresInitAve (); // reset counters for saved fields (costs very little
     191      dbExtractMeasuresInitAve (); // reset counters for saved fields (costs very little)
    192192
    193193      for (k = 0; (k < catalog.average[j].Nmeasure); k++, m++) {

  • trunk/Ohana/src/opihi/dvo/remote.c

    r33662 r33963  
    11# include "dvoshell.h"
    22# include <glob.h>
    3 # define MAX_PATH_LENGTH 1024
     3# define DVO_MAX_PATH 1024
    44
    55// functions to manage the remote hosts
    … …  
    88  int N;
    99
     10  int VERBOSE = FALSE;
     11  if ((N = get_argument (argc, argv, "-v"))) {
     12    remove_argument (N, &argc, argv);
     13    VERBOSE = TRUE;
     14  }
     15
    1016  if (argc < 2) {
    1117    gprint (GP_ERR, "USAGE: remote (command)\n");
    … …  
    1319    gprint (GP_ERR, "  options:\n");
    1420    gprint (GP_ERR, "  -v : verbose mode:\n");
     21    gprint (GP_ERR, "OR:    remote -reload (uniquer)\n");
    1522    return FALSE;
    1623  }
    … …  
    2330  // with a set of vectors to load.
    2431
    25   int VERBOSE = FALSE;
    26   if ((N = get_argument (argc, argv, "-v"))) {
     32  if ((N = get_argument (argc, argv, "-reload"))) {
    2733    remove_argument (N, &argc, argv);
    28     VERBOSE = TRUE;
     34    if (argc != 2) {
     35      gprint (GP_ERR, "USAGE: remote -reload (uniquer)\n");
     36      gprint (GP_ERR, " (uniquer) is the element in the middle of the results file\n");
     37      gprint (GP_ERR, " eg: dvo.results.XXXXX.YYYYY.fits\n");
     38      return FALSE;
     39    }
     40    int status = HostTableReloadResults (argv[1], VERBOSE);
     41    return status;
    2942  }
    3043

  • trunk/Ohana/src/opihi/dvo/skycoverage.c

    r33662 r33963  
    22
    33// enum to define possible modes
    4 enum {COVERAGE, DENSITY, MIN_UBERCAL, MIN_DMAG_SYS, MIN_MCAL, MAX_MCAL};
     4enum {COVERAGE, DENSITY, MIN_UBERCAL, MIN_DMAG_SYS, MIN_MCAL, MAX_MCAL, MIN_TIME, MAX_TIME};
    55
    66int wordhash (char *word);
    … …  
    2323  PhotCode *PhotcodeValue;
    2424
     25  time_t TimeReference;
     26  int TimeFormat;
     27
     28  GetTimeFormat (&TimeReference, &TimeFormat);
     29
    2530  WITH_MOSAIC = FALSE;
    2631  if ((N = get_argument (argc, argv, "+mosaic"))) {
    … …  
    7176    remove_argument (N, &argc, argv);
    7277    mode = MIN_UBERCAL;
     78  }
     79  if ((N = get_argument (argc, argv, "-min-time"))) {
     80    remove_argument (N, &argc, argv);
     81    mode = MIN_TIME;
     82  }
     83  if ((N = get_argument (argc, argv, "-max-time"))) {
     84    remove_argument (N, &argc, argv);
     85    mode = MAX_TIME;
    7386  }
    7487  if ((N = get_argument (argc, argv, "-min-dmag-sys"))) {
    … …  
    231244          case MIN_DMAG_SYS:
    232245          case MIN_MCAL:
    233             V[ys*Nx + xs] = 1E6;
     246          case MIN_TIME:
     247            V[ys*Nx + xs] = 1E9;
    234248            break;
    235249          case MAX_MCAL:
    236             V[ys*Nx + xs] = -1E6;
     250          case MAX_TIME:
     251            V[ys*Nx + xs] = -1E9;
    237252            break;
    238253        }
    … …  
    247262          case MIN_MCAL:
    248263          case MAX_MCAL:
     264          case MIN_TIME:
     265          case MAX_TIME:
    249266            V[ys*Nx + xs] = NAN;
    250267            break;
    … …  
    323340              V[ys*Nx + xs] = MAX(V[ys*Nx + xs], image[i].Mcal);
    324341              break;
     342            case MIN_TIME: {
     343              double timeVal = TimeValue (image[i].tzero, TimeReference, TimeFormat);
     344              V[ys*Nx + xs] = MIN(V[ys*Nx + xs], timeVal);
     345              break; }
     346            case MAX_TIME: {
     347              double timeVal = TimeValue (image[i].tzero, TimeReference, TimeFormat);
     348              V[ys*Nx + xs] = MAX(V[ys*Nx + xs], timeVal);
     349              break; }
    325350          }
    326351        }

  • trunk/Ohana/src/opihi/include/dvoshell.h

    r33662 r33963  
    9999dbValue      dbExtractImages        PROTO((Image *image, off_t Nimage, off_t N, dbField *field));
    100100
    101 int          HostTableLaunchJobs    PROTO((HostTable *table, char *basecmd, char *options));
     101int          HostTableLaunchJobs    PROTO((HostTable *table, char *basecmd, char *options, int VERBOSE));
    102102int          HostTableParallelOps   PROTO((int argc, char **argv, char *ResultFile, int Nelements, int VERBOSE));
    103 # endif
     103int          HostTableReloadResults PROTO((char *uniquer, int VERBOSE));
     104
     105# endif // DVOSHELL_H

  • trunk/Ohana/src/opihi/lib.shell/ListOps.c

    r33662 r33963  
    116116 
    117117  status = !strcmp (comm, "for");
     118  free (comm);
     119  return (status);
     120}
     121
     122int is_foreach_loop (char *line) {
     123
     124  int status;
     125  char *comm;
     126
     127  comm = thisword (line);
     128  if (comm == (char *) NULL) return (FALSE);
     129 
     130  status = !strcmp (comm, "foreach");
    118131  free (comm);
    119132  return (status);
    … …  
    266279  status |= is_macro_create (line);
    267280  status |= is_for_loop (line);
     281  status |= is_foreach_loop (line);
    268282  status |= is_list_data (line);
    269283  status |= is_loop (line);

  • trunk/Ohana/src/opihi/lib.shell/VectorIO.c

    r33662 r33963  
    2020  }
    2121  if (f == (FILE *) NULL) {
    22     gprint (GP_ERR, "can't open file for write\n");
     22    gprint (GP_ERR, "can't open file for write : %s\n", filename);
    2323    return (FALSE);
    2424  }

  • trunk/Ohana/src/opihi/lib.shell/startup.c

    r27998 r33963  
    138138    if (is_script) {
    139139      /* first argument in input script, rest are argv */
     140      set_str_variable ("SCRIPT_NAME", argv[1]);
    140141      list[Nlist] = strcreate (argv[1]);
    141142      Nlist ++;
Note: See TracChangeset for help on using the changeset viewer.