Index: /branches/pap/DataStore/scripts/dsgetfileset =================================================================== --- /branches/pap/DataStore/scripts/dsgetfileset (revision 28002) +++ /branches/pap/DataStore/scripts/dsgetfileset (revision 28003) @@ -17,5 +17,5 @@ use File::Basename qw( basename ); -my ($uri, $outdir, $timeout, $skip_checks, $no_proxy, $verbose); +my ($uri, $outdir, $timeout, $skip_checks, $unpack, $no_proxy, $verbose); GetOptions( @@ -24,4 +24,5 @@ 'timeout|t=s' => \$timeout, 'skip-checks' => \$skip_checks, + 'unpack' => \$unpack, 'no-proxy' => \$no_proxy, 'verbose|v' => \$verbose, @@ -89,4 +90,24 @@ exit $status; } + + if ($unpack) { + my $file_type = `file $outfile`; + if ($file_type =~ /FITS/) { + my $packed_file = "${outfile}.fz"; + rename $outfile, $packed_file + or die "failed to rename $outfile $packed_file"; + + # unpack the fits file and delete the input + # This works whether or not the file is compressed + $command = "funpack -D $packed_file"; + print "$command\n" if $verbose; + $rc = system $command; + if ($rc) { + my $status = $rc >> 8; + print STDERR "failed to funpack $outfile: rc: $rc status: $status\n"; + exit $status; + } + } + } } @@ -129,4 +150,8 @@ Do not load proxy environment variables. + +=item * --unpack + +Uncompress fits files if compressed Optional. Index: /branches/pap/DataStoreServer/scripts/dsprodtool =================================================================== --- /branches/pap/DataStoreServer/scripts/dsprodtool (revision 28002) +++ /branches/pap/DataStoreServer/scripts/dsprodtool (revision 28003) @@ -38,4 +38,5 @@ my $dsroot; my $dbname; +my $dbserver; my $dbpass; @@ -61,4 +62,5 @@ 'ps7=s' => \$ps7, 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, 'dsroot=s' => \$dsroot, ) or pod2usage(2); @@ -124,5 +126,6 @@ if ($err); -my $dbserver = metadataLookupStr($siteConfig, 'DS_DBSERVER'); +$dbserver = metadataLookupStr($siteConfig, 'DS_DBSERVER') + unless defined $dbserver; $dbname = metadataLookupStr($siteConfig, 'DS_DBNAME') unless defined $dbname; Index: /branches/pap/Ohana/src/addstar/src/find_matches_closest_refstars.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/find_matches_closest_refstars.c (revision 28002) +++ /branches/pap/Ohana/src/addstar/src/find_matches_closest_refstars.c (revision 28003) @@ -159,5 +159,8 @@ /** in replace mode, search for entry and replace values M, dM, R, D */ // XXX this fails for unsorted catalogs, right? - if (options.replace && replace_match (&catalog[0].average[n], catalog[0].measure, stars[N])) continue; + if (options.replace && replace_match (&catalog[0].average[n], catalog[0].measure, stars[N])) { + i++; + continue; + } /* make sure there is space for next entry */ Index: /branches/pap/Ohana/src/dvomerge/src/merge_catalogs_old.c =================================================================== --- /branches/pap/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 28002) +++ /branches/pap/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 28003) @@ -10,5 +10,5 @@ int merge_catalogs_old (SkyRegion *region, Catalog *output, Catalog *input, double RADIUS) { - off_t i, j, n, N, Nin, offset, J, Jmin, status, Nstars; + off_t i, j, Nin, offset, J, Jmin, status, Nstars; double RADIUS2, Rmin, Rin, Din; double *X1, *Y1, *X2, *Y2; @@ -152,6 +152,6 @@ /*** a match is found, add to average, measure ***/ Nmatch ++; - n = N2[Jmin]; - N = N1[i]; + off_t n = N2[Jmin]; + off_t N = N1[i]; /* make sure there is space for next Nmeasure entries */ @@ -223,4 +223,6 @@ /* these new entries are all written out in UPDATE mode */ for (i = 0; i < Nstars; i++) { + off_t N = N1[i]; + /* make sure there is space for next entry */ if (Nmeas + input[0].average[N].Nmeasure >= NMEAS) { @@ -235,5 +237,4 @@ } - N = N1[i]; if (input[0].found[N] >= 0) continue; if (!IN_REGION (input[0].average[N].R, input[0].average[N].D)) continue; Index: /branches/pap/Ohana/src/getstar/src/MatchCoords.c =================================================================== --- /branches/pap/Ohana/src/getstar/src/MatchCoords.c (revision 28002) +++ /branches/pap/Ohana/src/getstar/src/MatchCoords.c (revision 28003) @@ -54,5 +54,5 @@ #ifdef notdef - printf("%d %8.1f %8.1f %s\n", i, x, y, dbImages[i].name); + printf("%ld %8.1f %8.1f %s\n", i, x, y, dbImages[i].name); #endif Index: /branches/pap/Ohana/src/kapa2/doc/mark.issues.txt =================================================================== --- /branches/pap/Ohana/src/kapa2/doc/mark.issues.txt (revision 28002) +++ /branches/pap/Ohana/src/kapa2/doc/mark.issues.txt (revision 28003) @@ -1,8 +1,13 @@ + +* option to set window margin +* option to add section outer box +* option to set limits based on associated image (and change dynamically...) +o need consistency of graph defaults : axis, tick, labels should come from kapa +o clear should erase current plot + -> The "plot" command needs to be fixed so that boundaries of contours do not cause it to draw excessive amounts of lines connecting unrelated contours. A specific "contour line type" would be useful. - --> DVO is not consistently placing the x/y axis labels, especially in -user-defined sections of the plot window. +(-pt 100?) -> DVO does not plot the end-points of histograms properly (e.g., the @@ -13,3 +18,47 @@ relay output to the file in the same order. Also, until DVO is closed, the output does not appear in the file. +(fixed both issues) +-> DVO is not consistently placing the x/y axis labels, especially in +user-defined sections of the plot window. +(fixed) +* box -labelpad -labelpad[x,y] +labelpad[x,y] - + + +--- + + +1)You wanted me to remind you to update ~ipp-svn; this was so I could have +the latest version of DVO on-hand + +2)The plotting of axis labels remains inconsistent; depending on section +dimensions and the presence/absence of tick-marks, labels will change +their distance from their corresponding axis. This results in labels +falling off the screen or overlapping with other plot sections + +3)Plot section locations in the plot window shift if labels/tick-marks are +present or absent + +4)It would be useful to have an option for the box or section command that +causes the background to be opaque. This will allow plots to have insets. + +5)It would be useful to have a rectangle plotting command in order to +shade out regions on plots + +6)The limits command combined with box will sometimes produce tick-marks +with only the zero-point labeled, making the axes difficult to interpret + +7)Text is printed to a file at different times when using either the echo +or fprintf commands alongside the output command. Sometimes output is +printed to the file immediately, while other times it is printed only when +the file is closed for output. + +8)The use of ^C as an interrupt does not always work when a macro is doing +for loops. + +9)It would be useful for DVO to echo which line of a macro contains an +error, or at which line it was forced to stop. This would help debugging a +lot. + +10)The cgrid could use an option to print RA/DEC tick-mark labels so that +image plots would be more informative Index: /branches/pap/Ohana/src/kapa2/doc/notes.txt =================================================================== --- /branches/pap/Ohana/src/kapa2/doc/notes.txt (revision 28003) +++ /branches/pap/Ohana/src/kapa2/doc/notes.txt (revision 28003) @@ -0,0 +1,8 @@ + +2010.04.14 + + * one font per frame + * user-settable box padding + * user-settable label & axis-label padding + + Index: /branches/pap/Ohana/src/kapa2/include/prototypes.h =================================================================== --- /branches/pap/Ohana/src/kapa2/include/prototypes.h (revision 28002) +++ /branches/pap/Ohana/src/kapa2/include/prototypes.h (revision 28003) @@ -26,5 +26,5 @@ /* X drawing utilities */ -void DrawFrame PROTO((KapaGraphWidget *graph)); +int DrawFrame PROTO((KapaGraphWidget *graph)); int DrawObjects PROTO((KapaGraphWidget *graph)); void DrawLabels PROTO((KapaGraphWidget *graph)); @@ -39,6 +39,7 @@ void DrawXErrors PROTO((KapaGraphWidget *graph, Gobjects *objects)); void DrawYErrors PROTO((KapaGraphWidget *graph, Gobjects *objects)); -void DrawTick PROTO((int fx, int fy, int dfx, int dfy, int P, double min, double max, double value, int IsLabel, int IsMajor, int naxis)); +void DrawTick PROTO((Graphic *graphic, Axis *axis, int P, TickMarkData *tick, int naxis)); void AxisTickScale PROTO((Axis *axis, double *major, double *minor)); +TickMarkData *CreateAxisTicks PROTO((Axis *axis, int *nticks)); /* EventLoop */ @@ -47,4 +48,5 @@ int Reconfig PROTO((XEvent *event)); void Refresh PROTO((void)); +void DrawSectionBG PROTO((Graphic *graphic, Section *section)); /* CheckPipe */ @@ -61,4 +63,5 @@ int SetSection PROTO((int sock)); int ListSection PROTO((int sock)); +int SetSectionBG PROTO((int sock)); int MoveSection PROTO((int sock)); int DefineSection PROTO((int sock)); @@ -84,5 +87,5 @@ void FreeSection PROTO((Section *section)); void FreeSections PROTO((void)); -Section *AddSection PROTO((char *name, float x, float y, float dx, float dy)); +Section *AddSection PROTO((char *name, float x, float y, float dx, float dy, int bg)); int DelSection PROTO((char *name)); int GetSectionByName PROTO((char *name)); @@ -112,5 +115,5 @@ void PSXErrors PROTO((KapaGraphWidget *graph, Gobjects *objects, FILE *f)); void PSYErrors PROTO((KapaGraphWidget *graph, Gobjects *objects, FILE *f)); -void PSTick PROTO((FILE *f, double fx, double fy, double dfx, double dfy, int P, double min, double max, double value, int IsLabel, int IsMajor, int naxis)); +void PSTick PROTO((Graphic *graphic, Axis *axis, int P, TickMarkData *tick, int naxis, FILE *f)); void ClipLinePS PROTO((double x0, double y0, double x1, double y1, double X0, double Y1, double X1, double Y0, FILE *f)); int PSimage PROTO((KapaImageWidget *image, FILE *f)); @@ -131,5 +134,5 @@ void bDrawXErrors PROTO((KapaGraphWidget *graph, Gobjects *object)); void bDrawYErrors PROTO((KapaGraphWidget *graph, Gobjects *object)); -void bDrawTick PROTO((double fx, double fy, double dfx, double dfy, int P, double min, double max, double value, int IsLabel, int IsMajor, int naxis)); +void bDrawTick PROTO((Axis *axis, int P, TickMarkData *tick, int naxis)); void bDrawClipLine PROTO((double x0, double y0, double x1, double y1, double X0, double Y1, double X1, double Y0)); bDrawBuffer *bDrawIt PROTO((void)); Index: /branches/pap/Ohana/src/kapa2/include/structures.h =================================================================== --- /branches/pap/Ohana/src/kapa2/include/structures.h (revision 28002) +++ /branches/pap/Ohana/src/kapa2/include/structures.h (revision 28003) @@ -137,8 +137,16 @@ double min, max; char isaxis, areticks, islabel, islog; + double pad, labelPad, ticktextPad; double fx, dfx, fy, dfy; /* axis location on graphic */ double lweight; int color; } Axis; + +// structure to describe the ticks for an axis +typedef struct { + double value; + int IsMajor; + int IsLabel; +} TickMarkData; // a single graph in the display window @@ -210,4 +218,5 @@ float x, y; float dx, dy; + int bg; char *name; } Section; Index: /branches/pap/Ohana/src/kapa2/src/CheckPipe.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/CheckPipe.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/CheckPipe.c (revision 28003) @@ -221,4 +221,10 @@ } + if (!strcmp (word, "BSEC")) { + status = SetSectionBG (sock); + KiiSendCommand (sock, 4, "DONE"); + FINISHED (TRUE); + } + if (!strcmp (word, "FONT")) { status = SetFont (sock); Index: /branches/pap/Ohana/src/kapa2/src/DefineSection.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/DefineSection.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/DefineSection.c (revision 28003) @@ -4,10 +4,10 @@ int DefineSection (int sock) { - int N, MoveSection; + int N, MoveSection, bg; char name[128]; double x, y, dx, dy; Section *section; - KiiScanMessage (sock, "%s %lf %lf %lf %lf", name, &x, &y, &dx, &dy); + KiiScanMessage (sock, "%s %lf %lf %lf %lf %d", name, &x, &y, &dx, &dy, &bg); MoveSection = FALSE; @@ -15,5 +15,5 @@ N = GetSectionByName (name); if (N < 0) { - section = AddSection (name, x, y, dx, dy); + section = AddSection (name, x, y, dx, dy, bg); MoveSection = TRUE; } else { @@ -30,4 +30,5 @@ section[0].dx = dx; section[0].dy = dy; + section[0].bg = bg; } Index: /branches/pap/Ohana/src/kapa2/src/DrawFrame.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/DrawFrame.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/DrawFrame.c (revision 28003) @@ -1,90 +1,66 @@ # include "Ximage.h" -# define MIN_RANGE 1e-10 + # define DrawLine(X,Y,DX,DY) (XDrawLine (graphic->display, graphic->window, graphic->gc, (int)(X), (int)(Y), (int)(X+DX), (int)(Y+DY))) -static Graphic *graphic; - -void DrawFrame (KapaGraphWidget *graph) { - - int i, fx, fy, dfx, dfy, P, IsLabel, IsMajor; - double range, major, minor, first, next, lweight; +int DrawFrame (KapaGraphWidget *graph) { + + int i, j, Nticks, P; + double fx, fy, dfx, dfy, lweight; + Graphic *graphic; + TickMarkData *ticks; + int color; graphic = GetGraphic(); - /* each axis is drawn independently, but ticks and labels are placed according to - perpendicular distance. */ + /* each axis is drawn independently, but ticks and labels are placed according to perpendicular distance. */ for (i = 0; i < 4; i++) { - fx = graph[0].axis[i].fx; - fy = graph[0].axis[i].fy; + fx = graph[0].axis[i].fx; + fy = graph[0].axis[i].fy; dfx = graph[0].axis[i].dfx; dfy = graph[0].axis[i].dfy; - P = hypot ((double)graph[0].axis[(i+1)%2].dfx, (double)graph[0].axis[(i+1)%2].dfy); + P = hypot (graph[0].axis[(i+1)%2].dfx, graph[0].axis[(i+1)%2].dfy); lweight = MAX (0, MIN (10, graph[0].axis[i].lweight)); + color = MAX (0, MIN (15, graph[0].axis[i].color)); + XSetLineAttributes (graphic->display, graphic->gc, lweight, LineSolid, CapNotLast, JoinMiter); - - graph[0].axis[i].color = MAX (0, MIN (15, graph[0].axis[i].color)); - - XSetForeground (graphic->display, graphic->gc, graphic->color[graph[0].axis[i].color]); - DrawRotTextInit (graphic->display, graphic->window, graphic->gc, graphic->color[graph[0].axis[i].color], graphic->back); - - // fprintf (stderr, "axis: %d, color: %d, %ld, weight: %f\n", i, graph[0].axis[i].color, graphic->color[graph[0].axis[i].color], lweight); + XSetForeground (graphic->display, graphic->gc, graphic->color[color]); + DrawRotTextInit (graphic->display, graphic->window, graphic->gc, graphic->color[color], graphic->back); if (graph[0].axis[i].isaxis) { DrawLine (fx, fy, dfx, dfy); - // fprintf (stderr, "%d : %d %d - %d %d\n", i, fx, fy, dfx, dfy); - // if (i == 0) { - // int j; - // for (j = fx; j < fx + dfx; j+= 10) { - // DrawLine (j, fy-10, 0, 20); - // } - // } } if (graph[0].axis[i].areticks) { - if (isnan(graph[0].axis[i].min) || isinf(graph[0].axis[i].min)) continue; - if (isnan(graph[0].axis[i].max) || isinf(graph[0].axis[i].max)) continue; - - range = graph[0].axis[i].max - graph[0].axis[i].min; - if (fabs(range) < MIN_RANGE) continue; - - if (fabs(range) < MIN_RANGE) { - range = (range < 0) ? -MIN_RANGE : +MIN_RANGE; + ticks = CreateAxisTicks (&graph[0].axis[i], &Nticks); + for (j = 0; j < Nticks; j++) { + DrawTick (graphic, &graph[0].axis[i], P, &ticks[j], i); } - - AxisTickScale (&graph[0].axis[i], &major, &minor); - - first = minor*((int)(graph[0].axis[i].min/minor)); - if ((range > 0) && (first < graph[0].axis[i].min)) { - first += minor; - } - if ((range < 0) && (first > graph[0].axis[i].min)) { - first -= minor; - } - for (next = first; ((range > 0) && (next <= graph[0].axis[i].max)) || ((range < 0) && (next >= graph[0].axis[i].max));) { - IsMajor = FALSE; - IsMajor |= (fabs((int)(next/major) - (next/major)) < 0.5*(minor/major)); - IsMajor |= (fabs((int)((next + 0.5*minor)/major) - (next/major)) < 0.5*(minor/major)); - IsMajor |= (fabs ((int)((next - 0.5*minor)/major) - (next/major)) < 0.5*(minor/major)); - IsLabel = (IsMajor && graph[0].axis[i].islabel); - DrawTick (fx, fy, dfx, dfy, P, graph[0].axis[i].min, graph[0].axis[i].max, next, IsLabel, IsMajor, i); - if (range > 0) - next += minor; - else - next -= minor; - } + FREE (ticks); } } -} - -void DrawTick (int fx, int fy, int dfx, int dfy, - int P, double min, double max, - double value, int IsLabel, int IsMajor, int naxis) { - - int x, y, dx, dy, pos, dir, fontsize; - double size, n; + return (TRUE); +} + +void DrawTick (Graphic *graphic, Axis *axis, int P, TickMarkData *tick, int naxis) { + + double x, y, dx, dy; + int pos, dir, fontsize; + double size, n, pad; char string[64], *fontname; - if (IsMajor) { + double fx = axis->fx; + double fy = axis->fy; + double dfx = axis->dfx; + double dfy = axis->dfy; + + double min = axis->min; + double max = axis->max; + + double value = tick->value; + + pos = size = 0; + + if (tick->IsMajor) { size = MAX (0.02, 7.0 / P); } else { @@ -92,10 +68,15 @@ } - n = P / sqrt ((double)(dfx*dfx + dfy*dfy)); + n = P / hypot(dfx, dfy); x = fx + (value-min)*dfx/(max - min); y = fy + (value-min)*dfy/(max - min); - dir = +1; - if ((naxis == 0) || (naxis == 1)) dir *= -1; + if ((naxis == 0) || (naxis == 2)) { + x = MAX(x, fx); + } + // y = MAX(y, fy); + // } + + dir = ((naxis == 0) || (naxis == 1)) ? -1 : +1; dx = dir*size*dfy*n; dy = dir*size*dfx*n; @@ -103,30 +84,28 @@ DrawLine (x, y, dx, dy); -# ifdef IN_AND_OUT_TICKS -# define GAP 0.03 -# else -# define GAP 0.01 -# endif - - if (IsLabel) { + if (tick->IsLabel) { + int xt, yt; + fontname = GetRotFont (&fontsize); - pos = 0; + + pad = !isnan(axis->ticktextPad) ? axis->ticktextPad*fontsize : fontsize + 4.0; + // pad = !isnan(axis->ticktextPad) ? axis->ticktextPad*fontsize : 0.8*fontsize + 1.0; /* temporarily assume rectilinear axes */ - if (naxis == 0) { dx = 0; dy = -dir*(0.8*fontsize + 1); pos = 1; } - if (naxis == 2) { dx = 0; dy = -dir*(0.8*fontsize + 1); pos = 7; } - - if (naxis == 1) { dy = 0; dx = -(0.8*fontsize + 1); pos = 3; } - if (naxis == 3) { dy = 0; dx = +(0.8*fontsize + 1); pos = 5; } - - x = fx + (value-min)*dfx/(max - min) + dx; - y = fy + (value-min)*dfy/(max - min) + dy; + if (naxis == 0) { dx = 0; dy = +1.0*pad; pos = 7; } + if (naxis == 2) { dx = 0; dy = -1.0*pad; pos = 1; } + + if (naxis == 1) { dy = 0; dx = -0.5*pad; pos = 3; } + if (naxis == 3) { dy = 0; dx = +0.5*pad; pos = 5; } + + xt = fx + (value-min)*dfx/(max - min) + dx; + yt = fy + (value-min)*dfy/(max - min) + dy; if (fabs(value/(max-min)) < 0.001) { value = 0.0; } sprintf (string, "%4g", value); - DrawRotText (x, y, string, pos, 0.0); - } - -} - + DrawRotText (xt, yt, string, pos, 0.0); + } +} + +# define MIN_RANGE 1e-10 void AxisTickScale (Axis *axis, double *major, double *minor) { @@ -192,2 +171,69 @@ } } + +TickMarkData *CreateAxisTicks (Axis *axis, int *nticks) { + + TickMarkData *ticks; + double range, major, minor, first, value, dPixels, overshoot; + int i, NTICKS, nPixels, done, ifirst; + + *nticks = 0; + + if (isnan(axis->min) || isinf(axis->min)) return NULL; + if (isnan(axis->max) || isinf(axis->max)) return NULL; + + range = axis->max - axis->min; + + // not sure why I skip out here instead of using the MIN_RANGE value + if (fabs(range) < MIN_RANGE) return NULL; + + if (fabs(range) < MIN_RANGE) { + range = (range < 0) ? -MIN_RANGE : +MIN_RANGE; + } + + // length of the axis in pixels + nPixels = hypot(axis->dfx, axis->dfy); + dPixels = nPixels / range; // axis pixel-scale + + AxisTickScale (axis, &major, &minor); + + // be a little generous + NTICKS = (int)(fabs(range / minor)) + 2; + ALLOCATE (ticks, TickMarkData, NTICKS); + + // value of the tick mark + ifirst = (axis->min < 0.0) ? (axis->min/minor - 0.05) : (axis->min/minor + 0.05); + first = minor*ifirst; + + // allow undershoot by 1 pixel + overshoot = (axis->min - first) * dPixels; + // if (overshoot > 0.5) { + // if (range > 0) { + // first += minor; + // } else { + // first -= minor; + // } + // } + + // loop to find the ticks + done = FALSE; + value = first; + for (i = 0; !done && (i < NTICKS); i++) { + ticks[i].IsMajor = FALSE; + ticks[i].IsMajor |= (fabs((int)(value/major) - (value/major)) < 0.5*(minor/major)); + ticks[i].IsMajor |= (fabs((int)((value + 0.5*minor)/major) - (value/major)) < 0.5*(minor/major)); + ticks[i].IsMajor |= (fabs((int)((value - 0.5*minor)/major) - (value/major)) < 0.5*(minor/major)); + ticks[i].IsLabel = (ticks[i].IsMajor && axis->islabel); + ticks[i].value = value; + if (range > 0) + value += minor; + else + value -= minor; + + done |= (range > 0) && (value > axis->max); + done |= (range < 0) && (value < axis->max); + } + + *nticks = i; + return (ticks); +} Index: /branches/pap/Ohana/src/kapa2/src/DrawLabels.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/DrawLabels.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/DrawLabels.c (revision 28003) @@ -34,6 +34,6 @@ switch (i) { case 0: pos = 7; break; + case 2: pos = 1; break; case 1: pos = 1; angle = -90; break; - case 2: pos = 1; break; case 3: pos = 1; angle = 90; break; case 4: pos = 2; break; Index: /branches/pap/Ohana/src/kapa2/src/EraseSections.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/EraseSections.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/EraseSections.c (revision 28003) @@ -8,5 +8,5 @@ FreeSections (); - AddSection ("default", 0.0, 0.0, 1.0, 1.0); + AddSection ("default", 0.0, 0.0, 1.0, 1.0, -1); if (USE_XWINDOW) XClearWindow (graphic->display, graphic->window); Index: /branches/pap/Ohana/src/kapa2/src/Graphs.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/Graphs.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/Graphs.c (revision 28003) @@ -18,5 +18,20 @@ graph[0].axis[i].lweight = 0.0; graph[0].axis[i].color = 0; + graph[0].axis[i].ticktextPad = NAN; + graph[0].axis[i].labelPad = NAN; + graph[0].axis[i].pad = NAN; } + graph[0].data.ticktextPad = NAN; + + graph[0].data.labelPadXm = NAN; + graph[0].data.labelPadXp = NAN; + graph[0].data.labelPadYm = NAN; + graph[0].data.labelPadYp = NAN; + + graph[0].data.padXm = NAN; + graph[0].data.padXp = NAN; + graph[0].data.padYm = NAN; + graph[0].data.padYp = NAN; + for (i = 0; i < 8; i++) { strcpy (graph[0].label[i].text, ""); @@ -37,6 +52,18 @@ graph[0].data.size = 1.0; // point size of 1.0 - graph[0].data.flipeast = 1; // +East = -X by default - graph[0].data.flipnorth = 0; // +North = +Y by default + graph[0].data.flipeast = TRUE; // +East = -X by default + graph[0].data.flipnorth = FALSE; // +North = +Y by default + + graph[0].data.coords.pc1_1 = graph[0].data.coords.pc2_2 = 1.0; + graph[0].data.coords.pc1_2 = graph[0].data.coords.pc2_1 = 0.0; + graph[0].data.coords.crval1 = 0.0; + graph[0].data.coords.crval2 = 0.0; + graph[0].data.coords.crpix1 = 0.0; + graph[0].data.coords.crpix2 = 0.0; + graph[0].data.coords.cdelt1 = graph[0].data.coords.cdelt2 = 1.0; + strcpy (graph[0].data.coords.ctype, "RA---LIN"); + strcpy (graph[0].data.axis, "2222"); + strcpy (graph[0].data.ticks, "2222"); + strcpy (graph[0].data.labels, "2222"); graph[0].Nobjects = 0; Index: /branches/pap/Ohana/src/kapa2/src/Layout.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/Layout.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/Layout.c (revision 28003) @@ -43,4 +43,4 @@ /* create basic section, empty of image or graph */ - section = AddSection ("default", 0.0, 0.0, 1.0, 1.0); + section = AddSection ("default", 0.0, 0.0, 1.0, 1.0, -1); } Index: /branches/pap/Ohana/src/kapa2/src/LoadFrame.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/LoadFrame.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/LoadFrame.c (revision 28003) @@ -3,7 +3,5 @@ int LoadFrame (int sock) { - int i, color; - char Axis[16], Labels[16], Ticks[16]; - double lweight; + int i; Section *section; KapaGraphWidget *graph; @@ -16,7 +14,5 @@ graph = section->graph; - KiiScanMessage (sock, "%lf %lf %lf %lf", - &graph[0].axis[0].min, &graph[0].axis[0].max, - &graph[0].axis[1].min, &graph[0].axis[1].max); + KapaScanGraphData (sock, &graph[0].data); graph[0].axis[3].min = graph[0].axis[1].min; @@ -25,14 +21,24 @@ graph[0].axis[2].max = graph[0].axis[0].max; - KiiScanMessage (sock, "%lf %d", &lweight, &color); - color = MAX (0, MIN (15, color)); + // XXX this is fragile + graph[0].data.color = MAX (0, MIN (15, graph[0].data.color)); - KiiScanMessage (sock, "%s %s %s", Axis, Labels, Ticks); + graph[0].axis[0].pad = graph[0].data.padXm; + graph[0].axis[1].pad = graph[0].data.padYm; + graph[0].axis[2].pad = graph[0].data.padXp; + graph[0].axis[3].pad = graph[0].data.padYp; + + graph[0].axis[0].labelPad = graph[0].data.labelPadXm; + graph[0].axis[1].labelPad = graph[0].data.labelPadYm; + graph[0].axis[2].labelPad = graph[0].data.labelPadXp; + graph[0].axis[3].labelPad = graph[0].data.labelPadYp; for (i = 0; i < 4; i++) { - graph[0].axis[i].lweight = lweight; - graph[0].axis[i].color = color; + graph[0].axis[i].lweight = graph[0].data.lweight; + graph[0].axis[i].color = graph[0].data.color; - switch (Axis[i]) { + graph[0].axis[i].ticktextPad = graph[0].data.ticktextPad; + + switch (graph[0].data.axis[i]) { case '0': graph[0].axis[i].isaxis = FALSE; @@ -45,5 +51,5 @@ break; } - switch (Ticks[i]) { + switch (graph[0].data.ticks[i]) { case '0': graph[0].axis[i].areticks = FALSE; @@ -56,5 +62,5 @@ break; } - switch (Labels[i]) { + switch (graph[0].data.labels[i]) { case '0': graph[0].axis[i].islabel = FALSE; @@ -70,6 +76,7 @@ SetSectionSizes (section); - if (USE_XWINDOW) DrawFrame (graph); - FlushDisplay (); + Refresh(); + // if (USE_XWINDOW) DrawFrame (graph); + // FlushDisplay (); /* XXX why did I do this??? */ Index: /branches/pap/Ohana/src/kapa2/src/PSFrame.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/PSFrame.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/PSFrame.c (revision 28003) @@ -4,9 +4,8 @@ int PSFrame (KapaGraphWidget *graph, FILE *f) { - int i, P, IsLabel, IsMajor, fontsize; - double fx, fy, dfx, dfy, range, major, minor, first, next; - char *fontname; + int i, Nticks, P; + double fx, fy, dfx, dfy, lweight; Graphic *graphic; - float weight; + TickMarkData *ticks; bDrawColor color; @@ -14,60 +13,54 @@ /* each axis is drawn independently */ - fontname = GetRotFont (&fontsize); fprintf (f, "1 setlinewidth\n"); for (i = 0; i < 4; i++) { - fx = graph[0].axis[i].fx; - fy = graphic->dy - graph[0].axis[i].fy; + fx = graph[0].axis[i].fx; + fy = graphic->dy - graph[0].axis[i].fy; dfx = graph[0].axis[i].dfx; dfy = -graph[0].axis[i].dfy; - P = hypot ((double)graph[0].axis[(i+1)%2].dfx, (double)graph[0].axis[(i+1)%2].dfy); + P = hypot (graph[0].axis[(i+1)%2].dfx, graph[0].axis[(i+1)%2].dfy); - weight = MAX (0, MIN (10, graph[0].axis[i].lweight)); + lweight = MAX (0, MIN (10, graph[0].axis[i].lweight)); color = MAX (0, MIN (15, graph[0].axis[i].color)); - fprintf (f, "%.1f setlinewidth\n", weight); + + fprintf (f, "%.1f setlinewidth\n", lweight); fprintf (f, "%s setrgbcolor\n", KapaColorRGBString(color)); + // no need to init rot font - if (graph[0].axis[i].isaxis) { DrawLine (fx, fy, dfx, dfy); } + if (graph[0].axis[i].isaxis) { + DrawLine (fx, fy, dfx, dfy); + } if (graph[0].axis[i].areticks) { - if (isnan(graph[0].axis[i].min) || isinf(graph[0].axis[i].min)) continue; - if (isnan(graph[0].axis[i].max) || isinf(graph[0].axis[i].max)) continue; - - range = graph[0].axis[i].max - graph[0].axis[i].min; - AxisTickScale (&graph[0].axis[i], &major, &minor); - - first = minor*((int)(graph[0].axis[i].min/minor)); - if ((range > 0) && (first < graph[0].axis[i].min)) { - first += minor; + ticks = CreateAxisTicks (&graph[0].axis[i], &Nticks); + for (i = 0; i < Nticks; i++) { + PSTick (graphic, &graph[0].axis[i], P, &ticks[i], i, f); } - if ((range < 0) && (first > graph[0].axis[i].min)) { - first -= minor; - } - for (next = first; ((range > 0) && (next <= graph[0].axis[i].max)) || ((range < 0) && (next >= graph[0].axis[i].max));) { - IsMajor = FALSE; - IsMajor |= (fabs((int)(next/major) - (next/major)) < 0.5*(minor/major)); - IsMajor |= (fabs ((int)((next + 0.5*minor)/major) - (next/major)) < 0.5*(minor/major)); - IsMajor |= (fabs ((int)((next - 0.5*minor)/major) - (next/major)) < 0.5*(minor/major)); - IsLabel = (IsMajor && graph[0].axis[i].islabel); - PSTick (f, fx, fy, dfx, dfy, P, graph[0].axis[i].min, graph[0].axis[i].max, next, IsLabel, IsMajor, i); - if (range > 0) - next += minor; - else - next -= minor; - } - } + FREE (ticks); + } } return (TRUE); } -void PSTick (FILE *f, double fx, double fy, double dfx, double dfy, int P, double min, double max, double value, int IsLabel, int IsMajor, int naxis) { +void PSTick (Graphic *graphic, Axis *axis, int P, TickMarkData *tick, int naxis, FILE *f) { + double x, y, dx, dy; int pos, dir, fontsize; - double size, n, x, y, dx, dy; + double size, n, pad; char string[64], *fontname; + + double fx = axis->fx; + double fy = graphic->dy - axis->fy; + double dfx = axis->dfx; + double dfy = -axis->dfy; + + double min = axis->min; + double max = axis->max; + + double value = tick->value; pos = size = 0; - if (IsMajor) { + if (tick->IsMajor) { size = MAX (0.02, 7.0 / P); } else { @@ -75,10 +68,9 @@ } - n = P / sqrt ((double)(dfx*dfx + dfy*dfy)); + n = P / hypot(dfx, dfy); x = fx + (value-min)*dfx/(max - min); y = fy + (value-min)*dfy/(max - min); - dir = -1; - if ((naxis == 0) || (naxis == 1)) dir *= -1; + dir = ((naxis == 0) || (naxis == 1)) ? -1 : +1; dx = dir*size*dfy*n; dy = dir*size*dfx*n; @@ -86,19 +78,23 @@ DrawLine (x, y, dx, dy); - if (IsLabel) { + if (tick->IsLabel) { + int xt, yt; + fontname = GetRotFont (&fontsize); + pad = !isnan(axis->ticktextPad) ? axis->ticktextPad*fontsize : 0.8*fontsize + 1.0; + /* temporarily assume rectilinear axes */ - if (naxis == 0) { dx = 0; dy = -dir*(0.8*fontsize + 1); pos = 1; } - if (naxis == 2) { dx = 0; dy = -dir*(0.8*fontsize + 1); pos = 7; } + if (naxis == 0) { dx = 0; dy = +pad; pos = 1; } + if (naxis == 2) { dx = 0; dy = -pad; pos = 7; } - if (naxis == 1) { dy = 0; dx = -(0.8*fontsize + 1); pos = 3; } - if (naxis == 3) { dy = 0; dx = +(0.8*fontsize + 1); pos = 5; } + if (naxis == 1) { dy = 0; dx = -pad; pos = 3; } + if (naxis == 3) { dy = 0; dx = +pad; pos = 5; } - x = fx + (value-min)*dfx/(max - min) + dx; - y = fy + (value-min)*dfy/(max - min) + dy; - if (fabs(value) < 1e-14) { value = 0.0; } - sprintf (string, "%g", value); - PSRotText (f, (int)x, (int)y, string, pos, 0.0); + xt = fx + (value-min)*dfx/(max - min) + dx; + yt = fy + (value-min)*dfy/(max - min) + dy; + if (fabs(value) < 0.001) { value = 0.0; } + sprintf (string, "%4g", value); + PSRotText (f, xt, yt, string, pos, 0.0); } } Index: /branches/pap/Ohana/src/kapa2/src/Refresh.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/Refresh.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/Refresh.c (revision 28003) @@ -21,4 +21,5 @@ for (i = 0; i < Nsection; i++) { section = GetSectionByNumber (i); + DrawSectionBG (graphic, section); DrawImage (section->image); DrawGraph (section->graph); @@ -28,2 +29,25 @@ } +void DrawSectionBG (Graphic *graphic, Section *section) { + + int Ncolors; + int X0, Y0, dX, dY; + + if (section->bg == -1) return; + + Ncolors = KapaColormapSize(); + + if (section->bg < 0) return; + if (section->bg >= Ncolors) return; + + XSetForeground (graphic->display, graphic->gc, graphic->color[section->bg]); + + dY = graphic[0].dy * section[0].dy; + dX = graphic[0].dx * section[0].dx; + Y0 = graphic[0].dy - graphic[0].dy * section[0].y - dY; + X0 = graphic[0].dx * section[0].x; + // fprintf (stderr, "section: (%s) %d %d - %d %d\n", section[0].name, X0, Y0, dX, dY); + + XFillRectangle (graphic[0].display, graphic[0].window, graphic[0].gc, X0, Y0, dX, dY); + return; +} Index: /branches/pap/Ohana/src/kapa2/src/Sections.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/Sections.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/Sections.c (revision 28003) @@ -16,4 +16,5 @@ section[0].dx = 0; section[0].dy = 0; + section[0].bg = -1; section[0].name = NULL; @@ -42,5 +43,5 @@ } -Section *AddSection (char *name, float x, float y, float dx, float dy) { +Section *AddSection (char *name, float x, float y, float dx, float dy, int bg) { int N; @@ -61,4 +62,5 @@ sections[N][0].dx = dx; sections[N][0].dy = dy; + sections[N][0].bg = bg; ActiveSection = N; return (sections[N]); @@ -155,4 +157,22 @@ } +int SetSectionBG (int sock) { + + int N, background; + char name[128]; + + KiiScanMessage (sock, "%s %d", name, &background); + + N = GetSectionByName (name); + if (N < 0) { + fprintf (stderr, "section %s not found\n", name); + return (TRUE); + } + + sections[N][0].bg = background; + + return (TRUE); +} + // return TRUE even for nonsense cases to avoid quitting kapa int MoveSection (int sock) { Index: /branches/pap/Ohana/src/kapa2/src/SetGraphData.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/SetGraphData.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/SetGraphData.c (revision 28003) @@ -18,38 +18,18 @@ graph = section->graph; - // get graph data from client - KiiScanMessage (sock, "%d %d %d %d %d %d %lf %lf", - &graph[0].data.style, - &graph[0].data.ptype, - &graph[0].data.ltype, - &graph[0].data.etype, - &graph[0].data.ebar, - &graph[0].data.color, - &graph[0].data.lweight, - &graph[0].data.size); + KapaScanGraphData (sock, &graph[0].data); - KiiScanMessage (sock, "%lf %lf %lf %lf", - &graph[0].data.xmin, - &graph[0].data.xmax, - &graph[0].data.ymin, - &graph[0].data.ymax); + for (i = 0; i < 4; i++) { + graph[0].axis[i].ticktextPad = graph[0].data.ticktextPad; + } + graph[0].axis[0].labelPad = graph[0].data.labelPadXm; + graph[0].axis[1].labelPad = graph[0].data.labelPadYm; + graph[0].axis[2].labelPad = graph[0].data.labelPadXp; + graph[0].axis[3].labelPad = graph[0].data.labelPadYp; - KiiScanMessage (sock, "%f %f %f %f", - &graph[0].data.coords.pc1_1, &graph[0].data.coords.pc2_2, - &graph[0].data.coords.pc1_2, &graph[0].data.coords.pc2_1); - - KiiScanMessage (sock, "%d %d %s", - &graph[0].data.flipeast, &graph[0].data.flipnorth, - graph[0].data.coords.ctype); - - KiiScanMessage (sock, "%lf %lf %f %f %f %f", - &graph[0].data.coords.crval1, - &graph[0].data.coords.crval2, - &graph[0].data.coords.crpix1, - &graph[0].data.coords.crpix2, - &graph[0].data.coords.cdelt1, - &graph[0].data.coords.cdelt2); - - + graph[0].axis[0].pad = graph[0].data.padXm; + graph[0].axis[1].pad = graph[0].data.padYm; + graph[0].axis[2].pad = graph[0].data.padXp; + graph[0].axis[3].pad = graph[0].data.padYp; xmin = graph[0].data.xmin; @@ -90,29 +70,5 @@ graph = section->graph; - KiiSendMessage (sock, "%8d %d %d %d %d %d %f %f", - graph[0].data.style, - graph[0].data.ptype, graph[0].data.ltype, - graph[0].data.etype, graph[0].data.ebar, graph[0].data.color, - graph[0].data.lweight, graph[0].data.size); - - KiiSendMessage (sock, "%g %g %g %g", - graph[0].data.xmin, graph[0].data.xmax, - graph[0].data.ymin, graph[0].data.ymax); - - KiiSendMessage (sock, "%g %g %g %g", - graph[0].data.coords.pc1_1, graph[0].data.coords.pc2_2, - graph[0].data.coords.pc1_2, graph[0].data.coords.pc2_1); - - KiiSendMessage (sock, "%d %d %s", - graph[0].data.flipeast, graph[0].data.flipnorth, - graph[0].data.coords.ctype); - - KiiSendMessage (sock, "%g %g %g %g %g %g", - graph[0].data.coords.crval1, - graph[0].data.coords.crval2, - graph[0].data.coords.crpix1, - graph[0].data.coords.crpix2, - graph[0].data.coords.cdelt1, - graph[0].data.coords.cdelt2); + KapaSendGraphData (sock, &graph[0].data); return (TRUE); Index: /branches/pap/Ohana/src/kapa2/src/SetGraphSize.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/SetGraphSize.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/SetGraphSize.c (revision 28003) @@ -6,12 +6,16 @@ void SetGraphSize (Section *section) { - int fontsize, bump, Nc; - int textpad, textdY, WdY; - double PADx, PADy, Dx, Dy; - double PXm, PXp, PYm, PYp; + int i, Nticks; + int fontsize, Nc = 0; + int textpad, textdY, WdY; + int dXm, dXp, dYm, dYp; + double padXm, padXp, padYm, padYp; + double minPADx, maxPADx, minPADy; + double minPAD, maxPAD; double X0, Y0, dX, dY; char string[64], *fontname; KapaGraphWidget *graph; Graphic *graphic; + TickMarkData *ticks; if (section == NULL) return; @@ -22,20 +26,95 @@ fontname = GetRotFont (&fontsize); - PADx = MAX (graphic->dx / 20.0, fontsize); - PADy = MAX (graphic->dy / 20.0, fontsize); - Dx = graphic->dx - 2*PADx; - Dy = graphic->dy - 2*PADy; + minPAD = 1.0*fontsize + 0; + maxPAD = 2.0*fontsize + 4; - /* each graph has a padding PXm, PXp, PYm, PYp */ - PXm = (graph[0].axis[1].islabel) ? 4*fontsize : 0; - PXp = (graph[0].axis[3].islabel) ? 4*fontsize : 0; - PYm = (graph[0].axis[0].islabel) ? 4*fontsize : 0; - PYp = (graph[0].axis[2].islabel) ? 4*fontsize : 0; - + minPADx = 1.5*fontsize; + maxPADx = 2.5*fontsize + 4.0; + + minPADy = 1.5*fontsize; + + // dXm : offset to label from lower x-axis, dYp : offset to label from right y-axis, etc + // padXm : padding below x-axis, padYp : padding to right of y-axis, etc + + // offset from lower x-axis to labels and tick values + // these depend on (a) existence of tick labels and (b) auto-offset or not + if (isnan(graph[0].axis[0].labelPad)) { + dXm = (graph[0].axis[0].islabel) ? maxPAD : minPAD; + } else { + dXm = graph[0].axis[0].labelPad * fontsize; + } + if (isnan(graph[0].axis[0].pad)) { + padXm = graph[0].axis[0].islabel ? maxPADx : minPADx; + } else { + padXm = graph[0].axis[0].pad * fontsize; + } + + // offset from upper x-axis to labels and tick values + // these depend on (a) existence of tick labels and (b) auto-offset or not + if (isnan(graph[0].axis[2].labelPad)) { + dXp = (graph[0].axis[2].islabel) ? maxPAD : minPAD; + } else { + dXp = graph[0].axis[2].labelPad * fontsize; + } + if (isnan(graph[0].axis[2].pad)) { + padXp = graph[0].axis[2].islabel ? maxPADx : minPADx; + } else { + padXp = graph[0].axis[2].pad * fontsize; + } + + // offset from left y-axis to labels and tick values + // these depend on (a) existence of tick labels and (b) auto-offset or not. + // in the auto-offset case, offset depends on the width of the tick label strings + if (graph[0].axis[1].islabel && (isnan(graph[0].axis[1].labelPad) || isnan(graph[0].axis[1].pad))) { + /* check for the max size of the axis label */ + Nc = 0; + ticks = CreateAxisTicks (&graph[0].axis[1], &Nticks); + for (i = 0; i < Nticks; i++) { + if (!ticks[i].IsMajor) continue; + sprintf (string, "%g", ticks[i].value); + Nc = MAX (Nc, strlen (string)); + } + FREE(ticks); + } + if (isnan(graph[0].axis[1].labelPad)) { + dYm = (graph[0].axis[1].islabel) ? (0.7*(Nc + 1.5)*fontsize) : minPAD; + } else { + dYm = graph[0].axis[1].labelPad * fontsize; + } + if (isnan(graph[0].axis[1].pad)) { + padYm = graph[0].axis[1].islabel ? (0.7*(Nc + 2.5)*fontsize) : minPADy; + } else { + padYm = graph[0].axis[1].pad * fontsize; + } + + // offset from right y-axis to labels and tick values + // these depend on (a) existence of tick labels and (b) auto-offset or not. + // in the auto-offset case, offset depends on the width of the tick label strings + if (graph[0].axis[3].islabel && (isnan(graph[0].axis[3].labelPad) || isnan(graph[0].axis[3].pad))) { + Nc = 0; + ticks = CreateAxisTicks (&graph[0].axis[3], &Nticks); + for (i = 0; i < Nticks; i++) { + if (!ticks[i].IsMajor) continue; + sprintf (string, "%g", ticks[i].value); + Nc = MAX (Nc, strlen (string)); + } + FREE(ticks); + } + if (isnan(graph[0].axis[3].labelPad)) { + dYp = (graph[0].axis[3].islabel) ? (0.7*(Nc + 1.5)*fontsize) : minPAD; + } else { + dYp = graph[0].axis[3].labelPad * fontsize; + } + if (isnan(graph[0].axis[3].pad)) { + padYp = graph[0].axis[3].islabel ? (0.7*(Nc + 2.5)*fontsize) : minPADy; + } else { + padYp = graph[0].axis[3].pad * fontsize; + } + /* basic size of the graph in Xwindow coordinates */ - X0 = PADx + PXm + (Dx * section[0].x); - Y0 = PADy + PYm + (Dy * section[0].y); - dX = (Dx * section[0].dx) - PXp - PXm; - dY = (Dy * section[0].dy) - PYp - PYm; + X0 = graphic[0].dx * section[0].x + padYm; + Y0 = graphic[0].dy * section[0].y + padXm; + dX = graphic[0].dx * section[0].dx - padYm - padYp; + dY = graphic[0].dy * section[0].dy - padXm - padXp; // if we are tied to an image, make mods as needed @@ -47,17 +126,17 @@ switch (section->image->location) { case 1: - Y0 = graphic[0].dy * section[0].y + 2*PAD1 + WdY + 2; // tied to image in Y - dY = graphic[0].dy * section[0].dy - 5*PAD1 - WdY - COLORPAD + 1; - break; + Y0 = graphic[0].dy * section[0].y + 2*PAD1 + WdY + 2; // tied to image in Y + dY = graphic[0].dy * section[0].dy - 5*PAD1 - WdY - COLORPAD + 1; + break; case 3: - dY = graphic[0].dy * section[0].dy - 5*PAD1 - WdY - COLORPAD - PADy - PYm; - break; + dY = graphic[0].dy * section[0].dy - 5*PAD1 - WdY - COLORPAD - padYm - padYp; + break; case 2: - X0 = graphic[0].dx * section[0].x + 2*PAD1 + ZOOM_X; - dX = graphic[0].dx * section[0].dx - 3*PAD1 - ZOOM_X; - break; + X0 = graphic[0].dx * section[0].x + 2*PAD1 + ZOOM_X; + dX = graphic[0].dx * section[0].dx - 3*PAD1 - ZOOM_X; + break; case 4: - dX = graphic[0].dx * section[0].dx - 3*PAD1 - ZOOM_X - PADx - PXm; - break; + dX = graphic[0].dx * section[0].dx - 3*PAD1 - ZOOM_X - padXm - padXp; + break; } } @@ -68,5 +147,5 @@ graph[0].axis[0].dfx = dX; graph[0].axis[0].dfy = 0; - + graph[0].axis[1].fx = X0; graph[0].axis[1].fy = graphic->dy - Y0; @@ -84,45 +163,28 @@ graph[0].axis[3].dfy = -dY; - PADx = 0.8*fontsize + 2; - PADy = 3.0*fontsize + 4; + /* define locations of axis labels */ + graph[0].label[LABELLL].x = graph[0].axis[0].fx; + graph[0].label[LABELLL].y = graph[0].axis[0].fy + dXm; + graph[0].label[LABELX0].x = graph[0].axis[0].fx + 0.5*graph[0].axis[0].dfx; + graph[0].label[LABELX0].y = graph[0].axis[0].fy + dXm; + graph[0].label[LABELLR].x = graph[0].axis[0].fx + graph[0].axis[0].dfx; + graph[0].label[LABELLR].y = graph[0].axis[0].fy + dXm; - /* define locations of axis labels */ - graph[0].label[LABELX0].x = graph[0].axis[0].fx + 0.5*graph[0].axis[0].dfx; - bump = (graph[0].axis[0].islabel) ? PADy : PADx; - graph[0].label[LABELX0].y = graph[0].axis[0].fy + bump; - + graph[0].label[LABELUL].x = graph[0].axis[2].fx; + graph[0].label[LABELUL].y = graph[0].axis[2].fy - dXp; graph[0].label[LABELX1].x = graph[0].axis[2].fx + 0.5*graph[0].axis[2].dfx; - bump = (graph[0].axis[2].islabel) ? PADy : PADx; - graph[0].label[LABELX1].y = graph[0].axis[2].fy - bump; - - /* check for the max size of the axis label */ - sprintf (string, "%4g", graph[0].axis[1].min); - Nc = strlen (string); - sprintf (string, "%4g", graph[0].axis[1].max); - Nc = MAX (Nc, strlen (string)); + graph[0].label[LABELX1].y = graph[0].axis[2].fy - dXp; + graph[0].label[LABELUR].x = graph[0].axis[2].fx + graph[0].axis[2].dfx; + graph[0].label[LABELUR].y = graph[0].axis[2].fy - dXp; graph[0].label[LABELY0].y = graph[0].axis[1].fy + 0.5*graph[0].axis[1].dfy; - bump = (graph[0].axis[1].islabel) ? (0.8*Nc*fontsize + 1) : PADx; - graph[0].label[LABELY0].x = graph[0].axis[1].fx - bump; - - sprintf (string, "%4g", graph[0].axis[3].min); - Nc = strlen (string); - sprintf (string, "%4g", graph[0].axis[3].max); - Nc = MAX (Nc, strlen (string)); + graph[0].label[LABELY0].x = graph[0].axis[1].fx - dYm; graph[0].label[LABELY1].y = graph[0].axis[3].fy + 0.5*graph[0].axis[3].dfy; - bump = (graph[0].axis[3].islabel) ? (0.8*Nc*fontsize + 1) : PADx; - graph[0].label[LABELY1].x = graph[0].axis[3].fx + bump; - + graph[0].label[LABELY1].x = graph[0].axis[3].fx + dYp; + + // fprintf (stderr, "section %s, %f,%f : %f,%f\n", section->name, X0, Y0, dX, dY); + /* these are wrong and have to be adjusted to sit in the corners */ - graph[0].label[LABELUL].x = graph[0].axis[2].fx - PADx; - graph[0].label[LABELUL].y = graph[0].axis[2].fy - PADx; - graph[0].label[LABELUR].x = graph[0].axis[2].fx + graph[0].axis[2].dfx + PADx; - graph[0].label[LABELUR].y = graph[0].axis[2].fy - PADx; - graph[0].label[LABELLL].x = graph[0].axis[0].fx - PADx; - graph[0].label[LABELLL].y = graph[0].axis[0].fy + PADx; - graph[0].label[LABELLR].x = graph[0].axis[0].fx + graph[0].axis[0].dfx + PADx; - graph[0].label[LABELLR].y = graph[0].axis[0].fy + PADx; - } Index: /branches/pap/Ohana/src/kapa2/src/bDrawFrame.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/bDrawFrame.c (revision 28002) +++ /branches/pap/Ohana/src/kapa2/src/bDrawFrame.c (revision 28003) @@ -1,14 +1,16 @@ # include "Ximage.h" +// bDrawLine is a function, not a macro like DrawLine int bDrawFrame (KapaGraphWidget *graph) { - int i, P, IsLabel, IsMajor, fontsize; - double fx, fy, dfx, dfy, range, major, minor, first, next; - char *fontname; - int weight; + int i, Nticks, P; + double fx, fy, dfx, dfy, lweight; + // Graphic graphic; is not needed + TickMarkData *ticks; bDrawColor color; + // don't need graphic, unlink DrawFrame + /* each axis is drawn independently */ - fontname = GetRotFont (&fontsize); for (i = 0; i < 4; i++) { fx = graph[0].axis[i].fx; @@ -16,9 +18,12 @@ dfx = graph[0].axis[i].dfx; dfy = graph[0].axis[i].dfy; - P = hypot ((double)graph[0].axis[(i+1)%2].dfx, (double)graph[0].axis[(i+1)%2].dfy); + P = hypot (graph[0].axis[(i+1)%2].dfx, graph[0].axis[(i+1)%2].dfy); - weight = MAX (0, MIN (10, graph[0].axis[i].lweight)); + lweight = MAX (0, MIN (10, graph[0].axis[i].lweight)); color = MAX (0, MIN (15, graph[0].axis[i].color)); - bDrawSetStyle (color, weight, 0); + + bDrawSetStyle (color, lweight, 0); + // function about sets solor and weight + // bDrawRotTextInit does not exist if (graph[0].axis[i].isaxis) { @@ -27,30 +32,9 @@ if (graph[0].axis[i].areticks) { - if (isnan(graph[0].axis[i].min) || isinf(graph[0].axis[i].min)) continue; - if (isnan(graph[0].axis[i].max) || isinf(graph[0].axis[i].max)) continue; - - range = graph[0].axis[i].max - graph[0].axis[i].min; - - AxisTickScale (&graph[0].axis[i], &major, &minor); - - first = minor*((int)(graph[0].axis[i].min/minor)); - if ((range > 0) && (first < graph[0].axis[i].min)) { - first += minor; + ticks = CreateAxisTicks (&graph[0].axis[i], &Nticks); + for (i = 0; i < Nticks; i++) { + bDrawTick (&graph[0].axis[i], P, &ticks[i], i); } - if ((range < 0) && (first > graph[0].axis[i].min)) { - first -= minor; - } - for (next = first; ((range > 0) && (next <= graph[0].axis[i].max)) || ((range < 0) && (next >= graph[0].axis[i].max));) { - IsMajor = FALSE; - IsMajor |= (fabs((int)(next/major) - (next/major)) < 0.5*(minor/major)); - IsMajor |= (fabs ((int)((next + 0.5*minor)/major) - (next/major)) < 0.5*(minor/major)); - IsMajor |= (fabs ((int)((next - 0.5*minor)/major) - (next/major)) < 0.5*(minor/major)); - IsLabel = (IsMajor && graph[0].axis[i].islabel); - bDrawTick (fx, fy, dfx, dfy, P, graph[0].axis[i].min, graph[0].axis[i].max, next, IsLabel, IsMajor, i); - if (range > 0) - next += minor; - else - next -= minor; - } + FREE (ticks); } } @@ -58,13 +42,24 @@ } -void bDrawTick (double fx, double fy, double dfx, double dfy, int P, double min, double max, double value, int IsLabel, int IsMajor, int naxis) { +void bDrawTick (Axis *axis, int P, TickMarkData *tick, int naxis) { + double x, y, dx, dy; int pos, dir, fontsize; - double size, n, x, y, dx, dy; + double size, n, pad; char string[64], *fontname; + + double fx = axis->fx; + double fy = axis->fy; + double dfx = axis->dfx; + double dfy = axis->dfy; + + double min = axis->min; + double max = axis->max; + + double value = tick->value; pos = size = 0; - if (IsMajor) { + if (tick->IsMajor) { size = MAX (0.02, 7.0 / P); } else { @@ -72,10 +67,9 @@ } - n = P / sqrt ((double)(dfx*dfx + dfy*dfy)); + n = P / hypot(dfx, dfy); x = fx + (value-min)*dfx/(max - min); y = fy + (value-min)*dfy/(max - min); - dir = +1; - if ((naxis == 0) || (naxis == 1)) dir *= -1; + dir = ((naxis == 0) || (naxis == 1)) ? -1 : +1; dx = dir*size*dfy*n; dy = dir*size*dfx*n; @@ -83,19 +77,23 @@ bDrawLine (x, y, x+dx, y+dy); - if (IsLabel) { + if (tick->IsLabel) { + int xt, yt; + fontname = GetRotFont (&fontsize); + pad = !isnan(axis->ticktextPad) ? axis->ticktextPad*fontsize : 0.8*fontsize + 1.0; + /* temporarily assume rectilinear axes */ - if (naxis == 0) { dx = 0; dy = -dir*(0.8*fontsize + 1); pos = 1; } - if (naxis == 2) { dx = 0; dy = -dir*(0.8*fontsize + 1); pos = 7; } + if (naxis == 0) { dx = 0; dy = +pad; pos = 1; } + if (naxis == 2) { dx = 0; dy = -pad; pos = 7; } - if (naxis == 1) { dy = 0; dx = -(0.8*fontsize + 1); pos = 3; } - if (naxis == 3) { dy = 0; dx = +(0.8*fontsize + 1); pos = 5; } + if (naxis == 1) { dy = 0; dx = -pad; pos = 3; } + if (naxis == 3) { dy = 0; dx = +pad; pos = 5; } - x = fx + (value-min)*dfx/(max - min) + dx; - y = fy + (value-min)*dfy/(max - min) + dy; - if (fabs(value) < 1e-14) { value = 0.0; } - sprintf (string, "%g", value); - bDrawRotText ((int)x, (int)y, string, pos, 0.0); + xt = fx + (value-min)*dfx/(max - min) + dx; + yt = fy + (value-min)*dfy/(max - min) + dy; + if (fabs(value) < 0.001) { value = 0.0; } + sprintf (string, "%4g", value); + bDrawRotText (xt, yt, string, pos, 0.0); } } Index: /branches/pap/Ohana/src/libdvo/include/dvo.h =================================================================== --- /branches/pap/Ohana/src/libdvo/include/dvo.h (revision 28002) +++ /branches/pap/Ohana/src/libdvo/include/dvo.h (revision 28003) @@ -443,4 +443,5 @@ void sort_image_subset (Image *image, off_t *subset, off_t N); void sort_coords_index (double *X, double *Y, off_t *S, off_t N); +void sort_coords_indexonly (double *X, double *Y, off_t *S, off_t N); void sort_regions (SkyRegion *region, off_t N); Index: /branches/pap/Ohana/src/libdvo/src/coordops.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/coordops.c (revision 28002) +++ /branches/pap/Ohana/src/libdvo/src/coordops.c (revision 28003) @@ -353,5 +353,5 @@ dY = (dM*dLdX - dL*dMdX) * Do; - // fprintf (stderr, "X,Y + dX,dY : %f, %f : %f, %f\n", X, Y, dX, dY); + // fprintf (stderr, "Xo,Yo + dX,dY : %f, %f : %f, %f\n", Xo, Yo, dX, dY); Xo += dX; Index: /branches/pap/Ohana/src/libdvo/src/dvosorts.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/dvosorts.c (revision 28002) +++ /branches/pap/Ohana/src/libdvo/src/dvosorts.c (revision 28003) @@ -46,4 +46,19 @@ } +/* sort a coordinate pair (X,Y) and the associated index (S) */ +void sort_coords_indexonly (double *X, double *Y, off_t *S, off_t N) { + +# define SWAPFUNC(A,B){ off_t itmp; \ + itmp = S[A]; S[A] = S[B]; S[B] = itmp; \ +} +# define COMPARE(A,B)(X[S[A]] < X[S[B]]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + void sort_regions (SkyRegion *region, off_t N) { Index: /branches/pap/Ohana/src/libkapa/include/kapa.h =================================================================== --- /branches/pap/Ohana/src/libkapa/include/kapa.h (revision 28002) +++ /branches/pap/Ohana/src/libkapa/include/kapa.h (revision 28003) @@ -72,4 +72,5 @@ float dx; float dy; + int bg; } KapaSection; @@ -78,4 +79,7 @@ int style, ptype, ltype, etype, ebar, color; double lweight, size; + double ticktextPad; + double labelPadXm, labelPadYm, labelPadXp, labelPadYp; + double padXm, padXp, padYm, padYp; Coords coords; int flipeast, flipnorth; @@ -173,7 +177,10 @@ int KapaSelectSection (int fd, char *name); int KapaGetSection (int fd, char *name); +int KapaSectionBG (int fd, char *name, int bg); int KapaMoveSection (int fd, char *name, char *direction); int KapaSetGraphData (int fd, Graphdata *graphmode); int KapaGetGraphData (int fd, Graphdata *graphmode); +int KapaScanGraphData (int fd, Graphdata *graphmode); +int KapaSendGraphData (int fd, Graphdata *graphmode); int KapaSetImageData (int fd, KapaImageData *graphmode); int KapaGetImageData (int fd, KapaImageData *graphmode); Index: /branches/pap/Ohana/src/libkapa/src/KapaColors.c =================================================================== --- /branches/pap/Ohana/src/libkapa/src/KapaColors.c (revision 28002) +++ /branches/pap/Ohana/src/libkapa/src/KapaColors.c (revision 28003) @@ -37,4 +37,6 @@ } } + if (!strcasecmp (name, "none")) return (-1); + fprintf (stderr, "color may be one of:\n"); for (i = 0; i < N_KAPA_COLORS; i++) { Index: /branches/pap/Ohana/src/libkapa/src/KapaOpen.c =================================================================== --- /branches/pap/Ohana/src/libkapa/src/KapaOpen.c (revision 28002) +++ /branches/pap/Ohana/src/libkapa/src/KapaOpen.c (revision 28003) @@ -31,8 +31,8 @@ if (status == -1) { if (errno == EADDRINUSE) { - close (InitSocket); - Address[0].sin_port ++; - if (Address[0].sin_port > MY_PORT_MAX) exit (2); - goto retry_server; + close (InitSocket); + Address[0].sin_port ++; + if (Address[0].sin_port > MY_PORT_MAX) exit (2); + goto retry_server; } perror ("bind: "); @@ -69,4 +69,5 @@ /* do I need to clear rfds on each pass? */ + FD_ZERO(&rfds); FD_SET (InitSocket, &rfds); status = select (InitSocket + 1, &rfds, NULL, NULL, &wait); @@ -98,7 +99,7 @@ for (i = 0; i < Nvalid; i++) { - /* valid IP addresses may be machines (120.90.121.142) or + /* valid IP addresses may be machines (120.90.121.142) or class C networks (120.90.121.0) */ - + /* for machine, address must match */ if ((0xff & (VALID[i] >> 24)) != 0) { @@ -138,8 +139,8 @@ test = TRUE; test &= (status == 4); - test &= ((ip1 > 0) && (ip1 < 256)); - test &= ((ip2 > 0) && (ip2 < 256)); - test &= ((ip3 > 0) && (ip3 < 256)); - test &= ((ip4 >=0) && (ip4 < 256)); + test &= ((ip1 > 0) && (ip1 < 256)); + test &= ((ip2 > 0) && (ip2 < 256)); + test &= ((ip3 > 0) && (ip3 < 256)); + test &= ((ip4 >=0) && (ip4 < 256)); if (!test) { fprintf (stderr, "invalid IP address %s\n", string); @@ -205,5 +206,5 @@ // the client uses a BLOCKing socket by default - fcntl (InitSocket, F_SETFL, !O_NONBLOCK); + fcntl (InitSocket, F_SETFL, !O_NONBLOCK); return (InitSocket); } @@ -211,5 +212,5 @@ int KapaOpen (char *kapa_exec, char *kapa_name) { - // kapa_exec may be kapa://host, in which case we attempt to connect to an + // kapa_exec may be kapa://host, in which case we attempt to connect to an // already running kapa, or the program path, in which case we are supposed // to launch it locally, then connect to it. @@ -265,10 +266,10 @@ unlink (socket_name); - strcpy (Address.sun_path, socket_name); - Address.sun_family = AF_UNIX; - InitSocket = socket (AF_UNIX, SOCK_STREAM, 0); + strcpy (Address.sun_path, socket_name); + Address.sun_family = AF_UNIX; + InitSocket = socket (AF_UNIX, SOCK_STREAM, 0); status = bind (InitSocket, (struct sockaddr *) &Address, sizeof (Address)); status = listen (InitSocket, 1); - + if (name == NULL) { sprintf (temp, "%s -socket %s &", kapa_exec, socket_name); @@ -282,5 +283,5 @@ AddressLength = sizeof (Address); - fcntl (InitSocket, F_SETFL, O_NONBLOCK); + fcntl (InitSocket, F_SETFL, O_NONBLOCK); # define NTRY 500 @@ -297,5 +298,5 @@ // the client uses a BLOCKing socket by default - fcntl (fd, F_SETFL, !O_NONBLOCK); + fcntl (fd, F_SETFL, !O_NONBLOCK); return (fd); } @@ -317,5 +318,5 @@ // the server uses an unblocked socket - fcntl (sock, F_SETFL, O_NONBLOCK); + fcntl (sock, F_SETFL, O_NONBLOCK); unlink (sockpath); return (sock); Index: /branches/pap/Ohana/src/libkapa/src/KapaWindow.c =================================================================== --- /branches/pap/Ohana/src/libkapa/src/KapaWindow.c (revision 28002) +++ /branches/pap/Ohana/src/libkapa/src/KapaWindow.c (revision 28003) @@ -38,10 +38,7 @@ KiiSendCommand (fd, 4, "DBOX"); - KiiSendMessage (fd, "%12.6g %12.6g %12.6g %12.6g", - graphdata[0].xmin, graphdata[0].xmax, graphdata[0].ymin, graphdata[0].ymax); - - KiiSendMessage (fd, "%lf %d", graphdata[0].lweight, graphdata[0].color); - - KiiSendMessage (fd, "%s %s %s", graphdata[0].axis, graphdata[0].labels, graphdata[0].ticks); + + KapaSendGraphData (fd, graphdata); + KiiWaitAnswer (fd, "DONE"); return (TRUE); @@ -84,4 +81,5 @@ } +// XXX drop this function? : kapa is responsible for the graph initialization int KapaInitGraph (Graphdata *graphdata) { @@ -103,7 +101,20 @@ graphdata[0].flipeast = TRUE; graphdata[0].flipnorth = FALSE; + strcpy (graphdata[0].axis, "2222"); strcpy (graphdata[0].ticks, "2222"); strcpy (graphdata[0].labels, "2222"); + + graphdata[0].ticktextPad = NAN; + + graphdata[0].labelPadXm = NAN; + graphdata[0].labelPadXp = NAN; + graphdata[0].labelPadYm = NAN; + graphdata[0].labelPadYp = NAN; + + graphdata[0].padXm = NAN; + graphdata[0].padXp = NAN; + graphdata[0].padYm = NAN; + graphdata[0].padYp = NAN; return (TRUE); @@ -128,10 +139,11 @@ } -int KapaSetGraphData (int fd, Graphdata *data) { - - /* tell kapa to look for the incoming image */ - KiiSendCommand (fd, 4, "SSTY"); - +int KapaSendGraphData (int fd, Graphdata *data) { + /* send kapa the plot details */ + KiiSendMessage (fd, "%g %g %g %g", + data[0].xmin, data[0].xmax, + data[0].ymin, data[0].ymax); + KiiSendMessage (fd, "%8d %d %d %d %d %d %f %f", data[0].style, @@ -140,7 +152,12 @@ data[0].lweight, data[0].size); + KiiSendMessage (fd, "%g %g %g %g %g", + data[0].ticktextPad, + data[0].padXm, data[0].padXp, + data[0].padYm, data[0].padYp); + KiiSendMessage (fd, "%g %g %g %g", - data[0].xmin, data[0].xmax, - data[0].ymin, data[0].ymax); + data[0].labelPadXm, data[0].labelPadXp, + data[0].labelPadYm, data[0].labelPadYp); KiiSendMessage (fd, "%g %g %g %g", @@ -160,14 +177,16 @@ data[0].coords.cdelt2); - KiiWaitAnswer (fd, "DONE"); - return (TRUE); -} - -int KapaGetGraphData (int fd, Graphdata *data) { - - /* tell kapa to look for the incoming image */ - KiiSendCommand (fd, 4, "GSTY"); - + KiiSendMessage (fd, "%s %s %s", data[0].axis, data[0].ticks, data[0].labels); + + return (TRUE); +} + +int KapaScanGraphData (int fd, Graphdata *data) { + /* send kapa the plot details */ + KiiScanMessage (fd, "%lf %lf %lf %lf", + &data[0].xmin, &data[0].xmax, + &data[0].ymin, &data[0].ymax); + KiiScanMessage (fd, "%d %d %d %d %d %d %lf %lf", &data[0].style, @@ -176,7 +195,12 @@ &data[0].lweight, &data[0].size); + KiiScanMessage (fd, "%lf %lf %lf %lf %lf", + &data[0].ticktextPad, + &data[0].padXm, &data[0].padXp, + &data[0].padYm, &data[0].padYp); + KiiScanMessage (fd, "%lf %lf %lf %lf", - &data[0].xmin, &data[0].xmax, - &data[0].ymin, &data[0].ymax); + &data[0].labelPadXm, &data[0].labelPadXp, + &data[0].labelPadYm, &data[0].labelPadYp); KiiScanMessage (fd, "%f %f %f %f", @@ -196,8 +220,31 @@ &data[0].coords.cdelt2); + KiiScanMessage (fd, "%s %s %s", data[0].axis, data[0].ticks, data[0].labels); + // XXX at some point, we need to add polynomials and 2-level mosaic // astrometry here. data[0].coords.Npolyterms = 0; + + return (TRUE); +} + +int KapaSetGraphData (int fd, Graphdata *data) { + + /* tell kapa to look for the incoming image */ + KiiSendCommand (fd, 4, "SSTY"); + + KapaSendGraphData (fd, data); + + KiiWaitAnswer (fd, "DONE"); + return (TRUE); +} + +int KapaGetGraphData (int fd, Graphdata *data) { + + /* tell kapa to look for the incoming image */ + KiiSendCommand (fd, 4, "GSTY"); + + KapaScanGraphData (fd, data); KiiWaitAnswer (fd, "DONE"); @@ -312,10 +359,19 @@ KiiSendCommand (fd, 4, "DSEC"); - KiiSendMessage (fd, "%s %6.3f %6.3f %6.3f %6.3f", + KiiSendMessage (fd, "%s %6.3f %6.3f %6.3f %6.3f %3d", section[0].name, section[0].x, section[0].y, section[0].dx, - section[0].dy); + section[0].dy, + section[0].bg); + KiiWaitAnswer (fd, "DONE"); + return (TRUE); +} + +int KapaSectionBG (int fd, char *name, int bg) { + + KiiSendCommand (fd, 4, "BSEC"); + KiiSendMessage (fd, "%s %3d", name, bg); KiiWaitAnswer (fd, "DONE"); return (TRUE); Index: /branches/pap/Ohana/src/libkapa/src/KiiPicture.c =================================================================== --- /branches/pap/Ohana/src/libkapa/src/KiiPicture.c (revision 28002) +++ /branches/pap/Ohana/src/libkapa/src/KiiPicture.c (revision 28003) @@ -46,5 +46,5 @@ Nwrite = write (fd, image[0].data1d, size); if (Nwrite != size) { - fprintf (stderr, "error reading picture\n"); + fprintf (stderr, "error sending picture to kapa\n"); return (FALSE); } @@ -105,5 +105,5 @@ Nwrite = write (fd, outbuffer, size); if (Nwrite != size) { - fprintf (stderr, "error reading picture\n"); + fprintf (stderr, "error sending picture to kapa\n"); return (FALSE); } @@ -142,6 +142,6 @@ Nbytes = image[0].Nx*sizeof(float); Nwrite = write (fd, image[0].data2d[j], Nbytes); - if (Nwrite != size) { - fprintf (stderr, "error reading picture\n"); + if (Nwrite != Nbytes) { + fprintf (stderr, "error sending picture to kapa\n"); return (FALSE); } Index: /branches/pap/Ohana/src/opihi/cmd.basic/break.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.basic/break.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/cmd.basic/break.c (revision 28003) @@ -1,15 +1,16 @@ # include "basic.h" +// auto_break is currently a global int exec_break (int argc, char **argv) { int N, value; + if ((N = get_argument (argc, argv, "-h"))) goto usage; + if ((N = get_argument (argc, argv, "-help"))) goto usage; + if ((N = get_argument (argc, argv, "--help"))) goto usage; + if ((N = get_argument (argc, argv, "-auto"))) { remove_argument (N, &argc, argv); - value = -1; - if (!strcasecmp (argv[N], "on")) value = 1; - if (!strcasecmp (argv[N], "off")) value = 0; - if (value == -1) { - gprint (GP_ERR, "USAGE: break -auto [on / off]\n"); + if (N == argc) { if (auto_break) gprint (GP_ERR, "auto break on\n"); @@ -18,10 +19,17 @@ return (FALSE); } + value = -1; + if (!strcasecmp (argv[N], "on")) value = 1; + if (!strcasecmp (argv[N], "off")) value = 0; + if (value == -1) goto usage; auto_break = value; return (TRUE); } - + loop_break = TRUE; return (FALSE); - + +usage: + gprint (GP_ERR, "USAGE: break -auto [on / off]\n"); + return (FALSE); } Index: /branches/pap/Ohana/src/opihi/cmd.basic/opihi.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.basic/opihi.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/cmd.basic/opihi.c (revision 28003) @@ -3,8 +3,8 @@ int opihi_setmode (int argc, char **argv) { - int value; + OpihiVerboseMode value; if (argc < 2) { - gprint (GP_ERR, "USAGE: opihi (verbose) [on/off]\n"); + gprint (GP_ERR, "USAGE: opihi (verbose) [on/off/error]\n"); return (FALSE); } @@ -13,17 +13,30 @@ if (argc == 2) { value = get_verbose_shell(); - if (value) { - gprint (GP_ERR, "opihi verbose mode: on\n"); - } else { - gprint (GP_ERR, "opihi verbose mode: off\n"); - } + switch (value) { + case OPIHI_VERBOSE_OFF: + gprint (GP_ERR, "opihi verbose mode: off\n"); + break; + case OPIHI_VERBOSE_ON: + gprint (GP_ERR, "opihi verbose mode: on\n"); + break; + case OPIHI_VERBOSE_ERROR: + gprint (GP_ERR, "opihi verbose mode: error\n"); + break; + default: + fprintf (stderr, "impossible condition\n"); + abort(); + } return (TRUE); } if (!strcasecmp(argv[2], "on")) { - set_verbose_shell (TRUE); + set_verbose_shell (OPIHI_VERBOSE_ON); return (TRUE); } if (!strcasecmp(argv[2], "off")) { - set_verbose_shell (FALSE); + set_verbose_shell (OPIHI_VERBOSE_OFF); + return (TRUE); + } + if (!strcasecmp(argv[2], "error")) { + set_verbose_shell (OPIHI_VERBOSE_ERROR); return (TRUE); } Index: /branches/pap/Ohana/src/opihi/cmd.basic/run_for.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.basic/run_for.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/cmd.basic/run_for.c (revision 28003) @@ -7,5 +7,5 @@ int ThisList, depth, i, done, status, found, NLINES, j; double start, end, delta; - char *input; + char *input, *endptr; double value, sign; Macro loop; @@ -16,8 +16,24 @@ } - start = atof (argv[2]); - end = atof (argv[3]); + start = strtod (argv[2], &endptr); + if (*endptr) { + gprint (GP_ERR, "for loop starting value must be numerical (%s)\n", argv[2]); + return (FALSE); + } + + end = strtod (argv[3], &endptr); + if (*endptr) { + gprint (GP_ERR, "for loop ending value must be numerical (%s)\n", argv[3]); + return (FALSE); + } + delta = 1.0; - if (argc == 5) delta = atof (argv[4]); + if (argc == 5) { + delta = strtod (argv[4], &endptr); + if (*endptr) { + gprint (GP_ERR, "for loop delta value must be numerical (%s)\n", argv[4]); + return (FALSE); + } + } sign = SIGN(delta); Index: /branches/pap/Ohana/src/opihi/cmd.data/Makefile =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/Makefile (revision 28002) +++ /branches/pap/Ohana/src/opihi/cmd.data/Makefile (revision 28003) @@ -72,4 +72,5 @@ $(SRC)/lookup.$(ARCH).o \ $(SRC)/matrix.$(ARCH).o \ +$(SRC)/match2d.$(ARCH).o \ $(SRC)/mkrgb.$(ARCH).o \ $(SRC)/mcreate.$(ARCH).o \ Index: /branches/pap/Ohana/src/opihi/cmd.data/box.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/box.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/cmd.data/box.c (revision 28003) @@ -25,5 +25,5 @@ } - graphmode.lweight = 1; + // graphmode.lweight = 1; if ((N = get_argument (argc, argv, "-lw"))) { remove_argument (N, &argc, argv); @@ -32,5 +32,5 @@ } - graphmode.color = KapaColorByName ("black"); + // graphmode.color = KapaColorByName ("black"); if ((N = get_argument (argc, argv, "-c"))) { remove_argument (N, &argc, argv); @@ -40,5 +40,6 @@ } - strcpy (graphmode.ticks, "2222"); + // XXX need to get the current values from kapa + // strcpy (graphmode.ticks, "2222"); if ((N = get_argument (argc, argv, "-ticks"))) { remove_argument (N, &argc, argv); @@ -51,5 +52,5 @@ } - strcpy (graphmode.labels, "2222"); + // strcpy (graphmode.labels, "2222"); if ((N = get_argument (argc, argv, "-labels"))) { remove_argument (N, &argc, argv); @@ -62,5 +63,5 @@ } - strcpy (graphmode.axis, "2222"); + // strcpy (graphmode.axis, "2222"); if ((N = get_argument (argc, argv, "-axis"))) { remove_argument (N, &argc, argv); @@ -71,4 +72,72 @@ if ((graphmode.axis[i] != '0') && (graphmode.axis[i] != '1') && (graphmode.axis[i] != '2')) { goto usage; } } + } + + if ((N = get_argument (argc, argv, "-tickpad"))) { + remove_argument (N, &argc, argv); + graphmode.ticktextPad = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-labelpadx"))) { + remove_argument (N, &argc, argv); + graphmode.labelPadXm = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-labelpady"))) { + remove_argument (N, &argc, argv); + graphmode.labelPadYm = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "+labelpadx"))) { + remove_argument (N, &argc, argv); + graphmode.labelPadXp = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "+labelpady"))) { + remove_argument (N, &argc, argv); + graphmode.labelPadYp = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-labelpad"))) { + remove_argument (N, &argc, argv); + graphmode.labelPadXm = atof(argv[N]); + graphmode.labelPadXp = atof(argv[N]); + graphmode.labelPadYm = atof(argv[N]); + graphmode.labelPadYp = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-pad"))) { + remove_argument (N, &argc, argv); + graphmode.padXm = atof(argv[N]); + graphmode.padXp = atof(argv[N]); + graphmode.padYm = atof(argv[N]); + graphmode.padYp = atof(argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-xpad"))) { + remove_argument (N, &argc, argv); + graphmode.padXm = atof(argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "+xpad"))) { + remove_argument (N, &argc, argv); + graphmode.padXp = atof(argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-ypad"))) { + remove_argument (N, &argc, argv); + graphmode.padYm = atof(argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "+ypad"))) { + remove_argument (N, &argc, argv); + graphmode.padYp = atof(argv[N]); + remove_argument (N, &argc, argv); } Index: /branches/pap/Ohana/src/opihi/cmd.data/create.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/create.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/cmd.data/create.c (revision 28003) @@ -14,5 +14,6 @@ if ((argc != 5) && (argc != 4)) { - gprint (GP_ERR, "USAGE: create vector start end [delta]\n"); + gprint (GP_ERR, "USAGE: create vector start end [delta] [-int]\n"); + gprint (GP_ERR, " -int : resulting vector is integer type (delta must be integer)\n"); return (FALSE); } @@ -20,8 +21,9 @@ if ((vec = SelectVector (argv[1], ANYVECTOR, TRUE)) == NULL) return (FALSE); + delta = 1; start = atof (argv[2]); end = atof (argv[3]); - delta = 1; if (argc == 5) delta = atof (argv[4]); + if ((start == end) || (delta == 0)) { gprint (GP_ERR, "error in value: %f to %f, %f\n", start, end, delta); Index: /branches/pap/Ohana/src/opihi/cmd.data/fit1d.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/fit1d.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/cmd.data/fit1d.c (revision 28003) @@ -53,9 +53,9 @@ return (FALSE); } - REQUIRE_VECTOR_FLT (xvec, FALSE); - REQUIRE_VECTOR_FLT (yvec, FALSE); + CastVector (xvec, OPIHI_FLT); + CastVector (yvec, OPIHI_FLT); if (Weight) { - REQUIRE_VECTOR_FLT (dyvec, FALSE); + CastVector (dyvec, OPIHI_FLT); if (xvec[0].Nelements != dyvec[0].Nelements) { gprint (GP_ERR, "vectors must have same length\n"); Index: /branches/pap/Ohana/src/opihi/cmd.data/fit2d.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/fit2d.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/cmd.data/fit2d.c (revision 28003) @@ -64,10 +64,10 @@ return (FALSE); } - REQUIRE_VECTOR_FLT (xvec, FALSE); - REQUIRE_VECTOR_FLT (yvec, FALSE); - REQUIRE_VECTOR_FLT (zvec, FALSE); + CastVector (xvec, OPIHI_FLT); + CastVector (yvec, OPIHI_FLT); + CastVector (zvec, OPIHI_FLT); if (Weight) { - REQUIRE_VECTOR_FLT (dzvec, FALSE); + CastVector (dzvec, OPIHI_FLT); if (xvec[0].Nelements != dzvec[0].Nelements) { gprint (GP_ERR, "vectors must have same length\n"); Index: /branches/pap/Ohana/src/opihi/cmd.data/histogram.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/histogram.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/cmd.data/histogram.c (revision 28003) @@ -3,11 +3,18 @@ int histogram (int argc, char **argv) { - int i, bin, Nbins; + int i, N, bin, Nbins; opihi_int *OUT; opihi_flt start, end, delta; - Vector *xvec, *yvec; + Vector *xvec, *yvec, *range; + + range = NULL; + if ((N = get_argument (argc, argv, "-range"))) { + remove_argument (N, &argc, argv); + if ((range = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } if ((argc != 6) && (argc != 5)) { - gprint (GP_ERR, "USAGE: hist invec outvec start end [delta]\n"); + gprint (GP_ERR, "USAGE: hist invec outvec start end [delta] [-range range]\n"); return (FALSE); } @@ -31,4 +38,11 @@ if ((xvec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); if ((yvec = SelectVector (argv[2], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + if (range) { + ResetVector (range, OPIHI_FLT, Nbins); + for (i = 0; i < range[0].Nelements; i++) { + range[0].elements.Flt[i] = start + i*delta; + } + } ResetVector (yvec, OPIHI_INT, Nbins); Index: /branches/pap/Ohana/src/opihi/cmd.data/init.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/init.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/cmd.data/init.c (revision 28003) @@ -61,4 +61,5 @@ int lookup PROTO((int, char **)); int matrix PROTO((int, char **)); +int match2d PROTO((int, char **)); int mkrgb PROTO((int, char **)); int mcreate PROTO((int, char **)); @@ -201,4 +202,5 @@ {1, "iminterp", minterp, "interpolate image pixels"}, {1, "matrix", matrix, "matrix math operations"}, + {1, "match2d", match2d, "match 2 pairs of X,Y vectors and return matched indexes"}, {1, "mkrgb", mkrgb, "convert 3 images to rgb jpeg (use Kapa for better control)"}, {1, "mset", mset, "insert a vector in an image"}, Index: /branches/pap/Ohana/src/opihi/cmd.data/match2d.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/match2d.c (revision 28003) +++ /branches/pap/Ohana/src/opihi/cmd.data/match2d.c (revision 28003) @@ -0,0 +1,133 @@ +# include "data.h" +int find_matches2d (Vector *X1, Vector *Y1, Vector *X2, Vector *Y2, double Radius, Vector *index1, Vector *index2); + +// match2d (X1) (Y1) (X2) (Y2) (Radius) [-index1 (index1)] [-index2 (index2)] [-nomatch1 nomatch1] [-nomatch2 nomatch2] +// X1[Index1] <=> X2[Index2] (etc) +int match2d (int argc, char **argv) { + + int N; + double Radius; + char *endptr; + Vector *X1vec, *Y1vec, *X2vec, *Y2vec; + Vector *index1, *index2; + + if ((N = get_argument (argc, argv, "-index1"))) { + remove_argument (N, &argc, argv); + if ((index1 = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { + if ((index1 = SelectVector ("index1", ANYVECTOR, TRUE)) == NULL) return (FALSE); + } + + if ((N = get_argument (argc, argv, "-index2"))) { + remove_argument (N, &argc, argv); + if ((index2 = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { + if ((index2 = SelectVector ("index2", ANYVECTOR, TRUE)) == NULL) return (FALSE); + } + + if (argc != 6) { + gprint (GP_ERR, "USAGE: match2d X1 Y1 X2 Y2 Radius [-index1 (index1)] [-index2 (index2)] [-closest]\n"); + // gprint (GP_ERR, "USAGE: match2d X1 Y1 X2 Y2 Radius [-index1 (index1)] [-index2 (index2)] [-closest]\n"); + // gprint (GP_ERR, " if -closest is provided, index1 & index2 will have the same length as X1 and X2 (respectively)\n"); + // gprint (GP_ERR, " with either the index of the match or a value of -1 for non-matches\n"); + return (FALSE); + } + + if ((X1vec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((Y1vec = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((X2vec = SelectVector (argv[3], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((Y2vec = SelectVector (argv[4], OLDVECTOR, TRUE)) == NULL) return (FALSE); + + if (X1vec[0].Nelements != Y1vec[0].Nelements) { + gprint (GP_ERR, "X1 & Y1 vectors must have same length\n"); + return (FALSE); + } + if (X2vec[0].Nelements != Y2vec[0].Nelements) { + gprint (GP_ERR, "X2 & Y2 vectors must have same length\n"); + return (FALSE); + } + + REQUIRE_VECTOR_FLT (X1vec, FALSE); + REQUIRE_VECTOR_FLT (Y1vec, FALSE); + REQUIRE_VECTOR_FLT (X2vec, FALSE); + REQUIRE_VECTOR_FLT (Y2vec, FALSE); + + Radius = strtod (argv[5], &endptr); + if (*endptr) { + gprint (GP_ERR, "Radius must be numerical (%s)\n", argv[5]); + return (FALSE); + } + + find_matches2d (X1vec, Y1vec, X2vec, Y2vec, Radius, index1, index2); + + return (TRUE); +} + +// we are not defining a relative offset DX,DY for now +int find_matches2d (Vector *X1, Vector *Y1, Vector *X2, Vector *Y2, double Radius, Vector *index1, Vector *index2) { + + off_t i, j, first_j, I, J, *N1, *N2, Nmatch, NMATCH, DMATCH; + double dX, dY, dR, Radius2; + + NMATCH = MAX(MAX(0.01*X1->Nelements, 0.01*X2->Nelements), 100); + DMATCH = NMATCH; + + ResetVector (index1, OPIHI_INT, NMATCH); + ResetVector (index2, OPIHI_INT, NMATCH); + + ALLOCATE (N1, off_t, X1->Nelements); + ALLOCATE (N2, off_t, X2->Nelements); + + for (i = 0; i < X1->Nelements; i++) { N1[i] = i; } + for (i = 0; i < X2->Nelements; i++) { N2[i] = i; } + + sort_coords_indexonly (X1->elements.Flt, Y1->elements.Flt, N1, X1->Nelements); + sort_coords_indexonly (X2->elements.Flt, Y2->elements.Flt, N2, X2->Nelements); + + Radius2 = Radius*Radius; + + Nmatch = 0; + for (i = j = 0; (i < X1->Nelements) && (j < X2->Nelements);) { + I = N1[i]; + J = N2[j]; + + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + + if (dX <= -1.02*Radius) { i++; continue; } + if (dX >= +1.02*Radius) { j++; continue; } + + // look for all matches of list2() to list1(i) + first_j = j; + for (j = first_j; (dX > -1.02*Radius) && (j < X2->Nelements); j++) { + J = N2[j]; + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + dY = Y1->elements.Flt[I] - Y2->elements.Flt[J]; + dR = dX*dX + dY*dY; + if (dR < Radius2) { + index1->elements.Int[Nmatch] = I; + index2->elements.Int[Nmatch] = J; + + // XXX track matches 1 and 2 with internal vector, save new nomatch index vectors + // after this loop + + Nmatch ++; + if (Nmatch >= NMATCH) { + NMATCH += DMATCH; + REALLOCATE (index1->elements.Int, opihi_int, NMATCH); + REALLOCATE (index2->elements.Int, opihi_int, NMATCH); + } + } + } + j = first_j; + i++; + } + index1->Nelements = Nmatch; + index2->Nelements = Nmatch; + + free (N1); + free (N2); + + return (TRUE); +} Index: /branches/pap/Ohana/src/opihi/cmd.data/section.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/section.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/cmd.data/section.c (revision 28003) @@ -1,9 +1,9 @@ # include "data.h" -enum {NONE, LIST, UP, DOWN, TOP, BOTTOM, TOOL}; +enum {NONE, LIST, UP, DOWN, TOP, BOTTOM, TOOL, BG}; int section (int argc, char **argv) { - int N, action, kapa; + int N, action, kapa, background; Graphdata graphmode; KapaSection section; @@ -37,4 +37,16 @@ location = argv[N]; remove_argument (N, &argc, argv); + } + + background = -1; // invisible + if ((N = get_argument (argc, argv, "-bg"))) { + remove_argument (N, &argc, argv); + if (!strcasecmp (argv[N], "NONE")) { + background = -1; + } else { + background = KapaColorByName(argv[N]); + } + remove_argument (N, &argc, argv); + action = BG; } @@ -82,4 +94,8 @@ break; + case BG: + KapaSectionBG (kapa, argv[1], background); + break; + case TOOL: if (!strcmp(location, "-x")) { @@ -121,4 +137,5 @@ section.dx = atof (argv[4]); section.dy = atof (argv[5]); + section.bg = background; KapaSetSection (kapa, §ion); return (TRUE); Index: /branches/pap/Ohana/src/opihi/include/shell.h =================================================================== --- /branches/pap/Ohana/src/opihi/include/shell.h (revision 28002) +++ /branches/pap/Ohana/src/opihi/include/shell.h (revision 28003) @@ -15,4 +15,5 @@ typedef enum {GP_FILE, GP_BUFF} gpMode; typedef enum {GP_LOG, GP_ERR} gpDest; +typedef enum {OPIHI_VERBOSE_OFF, OPIHI_VERBOSE_ON, OPIHI_VERBOSE_ERROR} OpihiVerboseMode; typedef int CommandF (int argc, char **argv); Index: anches/pap/Ohana/src/opihi/lib.data/open_graph.c =================================================================== --- /branches/pap/Ohana/src/opihi/lib.data/open_graph.c (revision 28002) +++ (revision ) @@ -1,176 +1,0 @@ -# include "display.h" -# include "shell.h" -# define DEBUG 0 - -/* we have space for several kapa windows */ -# define NXGRAPH 5 - -static int Active; -static int Xgraph[NXGRAPH]; -static Graphdata graphdata[NXGRAPH]; - -void QuitGraph () { - - int i; - - for (i = 0; i < NXGRAPH; i++) { - KiiClose (Xgraph[i]); - } -} - -void InitGraph () { - - int i; - - Active = 0; - for (i = 0; i < NXGRAPH; i++) { - Xgraph[i] = -1; - - graphdata[i].xmin = graphdata[i].ymin = 0.0; - graphdata[i].xmax = graphdata[i].ymax = 1.0; - graphdata[i].style = graphdata[i].ptype = 0; - graphdata[i].ltype = graphdata[i].color = 0; - graphdata[i].etype = graphdata[i].ebar = 0; - graphdata[i].lweight = graphdata[i].size = 1.0; - - graphdata[i].coords.pc1_1 = graphdata[i].coords.pc2_2 = 1.0; - graphdata[i].coords.pc1_2 = graphdata[i].coords.pc2_1 = 0.0; - strcpy (graphdata[i].coords.ctype, "RA---LIN"); - graphdata[i].coords.crval1 = 0.0; - graphdata[i].coords.crval2 = 0.0; - graphdata[i].coords.crpix1 = 0.0; - graphdata[i].coords.crpix2 = 0.0; - graphdata[i].coords.cdelt1 = graphdata[i].coords.cdelt2 = 1.0; - graphdata[i].flipeast = TRUE; - graphdata[i].flipnorth = FALSE; - strcpy (graphdata[i].axis, "2222"); - strcpy (graphdata[i].ticks, "2222"); - strcpy (graphdata[i].labels, "2222"); - } -} - -/* set SIGPIPE to this function to close cleanly */ -void XGraphDead (int input) { - signal (SIGPIPE, XGraphDead); - gprint (GP_ERR, "kapa is dead, must restart\n"); - Xgraph[Active] = -1; -} - -/** start socketed connection */ -int open_graph (int N) { - - int fd; - char *kapa_exec, name[16]; - - kapa_exec = get_variable ("KAPA"); - if (kapa_exec == (char *) NULL) { - gprint (GP_ERR, "variable KAPA not found\n"); - return (FALSE); - } - - snprintf (name, 16, "[%d]", N); - fd = KiiOpen (kapa_exec, name); - free (kapa_exec); - - if (fd < 0) { - gprint (GP_ERR, "error starting kapa\n"); - return (FALSE); - } - - Xgraph[N] = fd; - return (TRUE); -} - -int close_graph (int N) { - - if (N < 0) return (FALSE); - if (N >= NXGRAPH) return (FALSE); - - KiiClose (Xgraph[N]); - Xgraph[N] = -1; - return (TRUE); -} - -/* return pointers for current Xgraph, set if desired, test, open if needed */ -int GetGraph (Graphdata *data, int *socket, int *N) { - - int i, n; - char buffer[70]; - - SetImageDevice (FALSE); - if (N == (int *) NULL) { - n = Active; - } else { - if (*N >= NXGRAPH) { - gprint (GP_ERR, "invalid Xgraph window %d\n", *N); - return (FALSE); - } - if (*N < 0) { - *N = n = Active; - } else { - Active = n = *N; - } - } - - /* test Xgraph[0], flush junk from pipe */ - signal (SIGPIPE, XGraphDead); - fcntl (Xgraph[n], F_SETFL, O_NONBLOCK); - for (i = 0; (read (Xgraph[n], buffer, 64) > 0) && (i < 20); i++); - fcntl (Xgraph[n], F_SETFL, !O_NONBLOCK); - - if (Xgraph[n] < 1) { - if (!open_graph(n)) { - return (FALSE); - } - } - - if (data != (Graphdata *) NULL) *data = graphdata[n]; - if (socket != (int *) NULL) *socket = Xgraph[n]; - - return (TRUE); - -} - -/* return pointers for given Xgraph, don't set or open */ -int GetGraphData (Graphdata *data, int *sock, int *N) { - - int n; - - if (N == (int *) NULL) { - n = Active; - } else { - if (*N >= NXGRAPH) { - gprint (GP_ERR, "invalid Xgraph window %d\n", *N); - return (FALSE); - } - if (*N < 0) { - n = Active; - } else { - n = *N; - } - } - - if (data != (Graphdata *) NULL) *data = graphdata[n]; - if (sock != (int *) NULL) *sock = Xgraph[n]; - - return (TRUE); - -} - -/* assign given values to current Xgraph */ -void SetGraph (Graphdata data) { - graphdata[Active] = data; -} - -/** internal tracking of current active device type **/ - -static int IsImage = FALSE; - -int GetCurrentDevice () { - return (IsImage); -} - -void SetImageDevice (int state) { - IsImage = state; -} - Index: anches/pap/Ohana/src/opihi/lib.data/open_image.c =================================================================== --- /branches/pap/Ohana/src/opihi/lib.data/open_image.c (revision 28002) +++ (revision ) @@ -1,133 +1,0 @@ -# include "display.h" -# include "shell.h" -# define DEBUG 0 - -/* we have space for several kii windows */ -# define NXIMAGE 5 - -static int Active; -static int Ximage[NXIMAGE]; -static char Ximbuffer[NXIMAGE][512]; -static double Xzero[NXIMAGE]; -static double Xrange[NXIMAGE]; - -void QuitImage () { - - int i; - - for (i = 0; i < NXIMAGE; i++) { - KiiClose (Ximage[i]); - } -} - -void InitImage () { - - int i; - - Active = 0; - for (i = 0; i < NXIMAGE; i++) { - Ximage[i] = -1; - Xzero[i] = 0; - Xrange[i] = 1024; - strcpy (Ximbuffer[i], "none"); - } -} - -/* set SIGPIPE to this function to close cleanly */ -void XImageDead (int input) { - signal (SIGPIPE, XImageDead); - gprint (GP_ERR, "kii is dead, must restart\n"); - Ximage[Active] = -1; -} - -/** start socketed connection */ -int open_image (int N) { - - int fd; - char *kii_exec, name[16]; - - kii_exec = get_variable ("KII"); - if (kii_exec == (char *) NULL) { - gprint (GP_ERR, "variable KII not found\n"); - return (FALSE); - } - - snprintf (name, 16, "[%d]", N); - fd = KiiOpen (kii_exec, name); - free (kii_exec); - - if (fd < 0) { - gprint (GP_ERR, "error starting kii\n"); - return (FALSE); - } - - Ximage[N] = fd; - return (TRUE); -} - -int close_image (int N) { - - if (N < 0) return (FALSE); - if (N >= NXIMAGE) return (FALSE); - - KiiClose (Ximage[N]); - Ximage[N] = -1; - return (TRUE); -} - -/* return pointers for current Ximage, set if desired, test, open if needed */ -int GetImage (int *socket, int *N) { - - int i, n; - char buffer[70]; - - SetImageDevice (TRUE); - if (N == (int *) NULL) { - n = Active; - } else { - if (*N >= NXIMAGE) { - gprint (GP_ERR, "invalid Ximage window %d\n", *N); - return (FALSE); - } - if (*N < 0) { - *N = n = Active; - } else { - Active = n = *N; - } - } - - /* test Ximage[0], flush junk from pipe */ - signal (SIGPIPE, XImageDead); - fcntl (Ximage[n], F_SETFL, O_NONBLOCK); - for (i = 0; (read (Ximage[n], buffer, 64) > 0) && (i < 20); i++); - fcntl (Ximage[n], F_SETFL, !O_NONBLOCK); - - if (Ximage[n] < 1) { - if (!open_image(n)) { - return (FALSE); - } - } - - if (socket != (int *) NULL) *socket = Ximage[n]; - - return (TRUE); - -} - -void SetImageName (char *name) { - strcpy (Ximbuffer[Active], name); -} - -char *GetImageName () { - return (Ximbuffer[Active]); -} - -// leave this information on kapa, request when needed? -void SetImageScale (double zero, double range) { - Xzero[Active] = zero; - Xrange[Active] = range; -} -void GetImageScale (double *zero, double *range) { - *zero = Xzero[Active]; - *range = Xrange[Active]; -} Index: /branches/pap/Ohana/src/opihi/lib.shell/command.c =================================================================== --- /branches/pap/Ohana/src/opihi/lib.shell/command.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/lib.shell/command.c (revision 28003) @@ -2,5 +2,5 @@ # define DEBUG 0 -static int VERBOSE_SHELL = FALSE; +static int VERBOSE_SHELL = OPIHI_VERBOSE_OFF; int command (char *line, char **outline, int VERBOSE) { @@ -10,6 +10,6 @@ Command *cmd; - rawline = NULL; - // rawline = strcreate (line); + // rawline = NULL; + rawline = strcreate (line); /* force a space between ! and first word: !ls becomes ! ls */ @@ -24,4 +24,9 @@ /* expand anything of the form fred[N] */ line = expand_vectors (line); /* line is freed here, new one allocated */ + + // print the line with the variables and vectors expanded, but before evaluating + // any in-line math expression + if (VERBOSE_SHELL == OPIHI_VERBOSE_ON) gprint (GP_ERR, "opihi: %s\n", line); + /* solve math expresions, assign variable, if needed */ line = parse (line); /* line is freed here, new one allocated */ @@ -31,6 +36,4 @@ *outline = line; - if (VERBOSE_SHELL) gprint (GP_ERR, "opihi: %s\n", line); - argv = parse_commands (line, &argc); if (argc == 0) { @@ -60,5 +63,5 @@ msg = get_variable_ptr ("ERRORMSG"); if (msg != (char *) NULL) gprint (GP_ERR, "%s\n", msg); - if (VERBOSE_SHELL) gprint (GP_ERR, "error on line: %s\n", rawline); + if (VERBOSE_SHELL != OPIHI_VERBOSE_OFF) gprint (GP_ERR, "error on line: %s\n", rawline); } Index: /branches/pap/Ohana/src/opihi/lib.shell/startup.c =================================================================== --- /branches/pap/Ohana/src/opihi/lib.shell/startup.c (revision 28002) +++ /branches/pap/Ohana/src/opihi/lib.shell/startup.c (revision 28003) @@ -124,7 +124,12 @@ if (LOAD_RC && !is_script) { rcfile = get_variable ("RCFILE"); - sprintf (line, "input %s/%s", home, rcfile); - status = command (line, &outline, TRUE); - if (outline != (char *) NULL) free (outline); + if (rcfile && check_file_access (rcfile, FALSE, FALSE, FALSE)) { + sprintf (line, "input %s/%s", home, rcfile); + status = command (line, &outline, TRUE); + if (outline != (char *) NULL) free (outline); + if (status) { + gprint (GP_LOG, "loaded file %s\n", rcfile); + } + } if (rcfile) free (rcfile); } Index: /branches/pap/Ohana/src/opihi/pantasks/Makefile =================================================================== --- /branches/pap/Ohana/src/opihi/pantasks/Makefile (revision 28002) +++ /branches/pap/Ohana/src/opihi/pantasks/Makefile (revision 28003) @@ -125,2 +125,6 @@ .PHONY: pantasks pantasks_client pantasks_server + +.PRECIOUS: $(SRC)/pantasks.c +.PRECIOUS: $(SRC)/pantasks_client.c +.PRECIOUS: $(SRC)/pantasks_server.c Index: /branches/pap/Ohana/src/opihi/pantasks/pantasks.c.in =================================================================== --- /branches/pap/Ohana/src/opihi/pantasks/pantasks.c.in (revision 28002) +++ /branches/pap/Ohana/src/opihi/pantasks/pantasks.c.in (revision 28003) @@ -18,4 +18,5 @@ InitBasic (); InitData (); + InitAstro (); InitPantasks (); Index: /branches/pap/Ohana/src/opihi/pantasks/pantasks_server.c.in =================================================================== --- /branches/pap/Ohana/src/opihi/pantasks/pantasks_server.c.in (revision 28002) +++ /branches/pap/Ohana/src/opihi/pantasks/pantasks_server.c.in (revision 28003) @@ -13,8 +13,10 @@ */ +void special_init (int *argc, char **argv); + /* program-dependent initialization */ int main (int argc, char **argv) { - char log_stdout[128], log_stderr[128]; + char log_stdout[1024], log_stderr[1024]; pthread_t JobsAndTasksThread; pthread_t clientsThread; @@ -23,11 +25,40 @@ SockAddress Address; - general_init (&argc, argv); + strcpy (log_stdout, "stdout"); + strcpy (log_stderr, "stderr"); - // define server output log files + special_init (&argc, argv); + + // + stdin = freopen ("/dev/zero", "r", stdin); + + // this block redirects the actual stderr, stdout steams to the output files if (VarConfig ("PANTASKS_SERVER_STDOUT", "%s", log_stdout) != NULL) { + if (strcmp (log_stdout, "stdout")) { + stdout = freopen (log_stdout, "a", stdout); + if (!stdout) { + fprintf (stderr, "failed to open %s for output\n", log_stdout); + exit (1); + } + } + } + if (VarConfig ("PANTASKS_SERVER_STDERR", "%s", log_stderr) != NULL) { + if (strcmp (log_stderr, "stderr")) { + stderr = freopen (log_stderr, "a", stderr); + if (!stderr) { + fprintf (stdout, "failed to open %s for error output\n", log_stderr); + exit (1); + } + } + } + + /* init for main (or only) thread */ + gprintInit (); + + // this block changes the target of the gprint output streams + if (strcmp (log_stdout, "stdout")) { gprintSetFileThisThread (GP_LOG, log_stdout); } - if (VarConfig ("PANTASKS_SERVER_STDERR", "%s", log_stderr) != NULL) { + if (strcmp (log_stderr, "stderr")) { gprintSetFileThisThread (GP_ERR, log_stderr); } @@ -41,4 +72,5 @@ InitBasic (); InitData (); + InitAstro (); InitPantasksServer (); @@ -105,2 +137,24 @@ void welcome () { } void program_init (int *argc, char **argv) {} + +// I need to do things in a different order here to have config / freopen / gprint sequenced correctly +void special_init (int *argc, char **argv) { + + /* init srand for rnd numbers elsewhere */ + long A, B; + A = time(NULL); + for (B = 0; A == time(NULL); B++); + srand48(B); + + /* set signals */ + signal (SIGINT, SIG_IGN); + + init_error (); + + /* load config data (.ptolemyrc) */ + if (!ConfigInit (argc, argv)) { + fprintf (stderr, "can't find (.ptolemyrc,ptolemy.rc) config file. some functions will be unavailable\n"); + } + + return; +} Index: /branches/pap/Ohana/src/relphot/src/StarOps.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/StarOps.c (revision 28002) +++ /branches/pap/Ohana/src/relphot/src/StarOps.c (revision 28003) @@ -225,4 +225,5 @@ } + // XXX make it optional to apply Mcal (do I need an extra field for advisory Mcal?) list[N] = Msys - catalog[i].measure[m].Mcal; dlist[N] = MAX (catalog[i].measure[m].dM, MIN_ERROR); Index: /branches/pap/Ohana/src/uniphot/Makefile =================================================================== --- /branches/pap/Ohana/src/uniphot/Makefile (revision 28002) +++ /branches/pap/Ohana/src/uniphot/Makefile (revision 28003) @@ -1,5 +1,5 @@ -default: uniphot +default: uniphot setphot help: - @echo "make options: uniphot (default)" +@echo "make options: uniphot setphot default help install default (uniphot setphot)" include ../../Makefile.System @@ -18,5 +18,6 @@ uniphot: $(BIN)/uniphot.$(ARCH) -install: $(DESTBIN)/uniphot +setphot: $(BIN)/setphot.$(ARCH) +install: $(DESTBIN)/uniphot $(DESTBIN)/setphot UNIPHOT = \ @@ -24,5 +25,5 @@ $(SRC)/initialize.$(ARCH).o \ $(SRC)/ConfigInit.$(ARCH).o \ -$(SRC)/args.$(ARCH).o \ +$(SRC)/args.$(ARCH).o \ $(SRC)/liststats.$(ARCH).o \ $(SRC)/load_images.$(ARCH).o \ @@ -39,2 +40,18 @@ $(UNIPHOT): $(INC)/uniphot.h $(BIN)/uniphot.$(ARCH): $(UNIPHOT) + +SETPHOT = \ +$(SRC)/setphot.$(ARCH).o \ +$(SRC)/initialize.$(ARCH).o \ +$(SRC)/ConfigInit.$(ARCH).o \ +$(SRC)/args.$(ARCH).o \ +$(SRC)/liststats.$(ARCH).o \ +$(SRC)/load_zpt_table.$(ARCH).o \ +$(SRC)/load_images.$(ARCH).o \ +$(SRC)/match_zpts_to_images.$(ARCH).o \ +$(SRC)/update_catalog_setphot.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o + +$(SETPHOT): $(INC)/uniphot.h +$(BIN)/setphot.$(ARCH): $(SETPHOT) Index: /branches/pap/Ohana/src/uniphot/include/uniphot.h =================================================================== --- /branches/pap/Ohana/src/uniphot/include/uniphot.h (revision 28002) +++ /branches/pap/Ohana/src/uniphot/include/uniphot.h (revision 28003) @@ -39,4 +39,10 @@ } Group; +typedef struct { + float zpt; + e_time time; + int found; +} ZptTable; + /* global variables set in parameter file */ char ImageCat[256]; @@ -44,4 +50,7 @@ char CATMODE[16]; /* raw, mef, split, mysql */ char CATFORMAT[16]; /* internal, elixir, loneos, panstarrs */ +char SKY_TABLE[256]; +int SKY_DEPTH; /** XXX EAM : depth of catalog tables, fix usage */ +char GSCFILE[256]; char STATMODE[64]; int VERBOSE; @@ -66,41 +75,50 @@ /***** prototypes ****/ -void ConfigInit PROTO((int *argc, char **argv)); -void DonePlotting PROTO((Graphdata *graphmode, int N)); -void JpegPlot PROTO((Graphdata *graphmode, int N, char *filename)); -void PSPlot PROTO((Graphdata *graphmode, int N, char *filename)); -void PlotLabel PROTO((char *string, int N)); -void PlotVector PROTO((int Npts, double *vect, int mode, int N)); -void PrepPlotting PROTO((int Npts, Graphdata *graphmode, int N)); -void XDead PROTO((void)); -int args PROTO((int argc, char **argv)); -void dumpresult PROTO((void)); -Group *find_image_sgroups PROTO((FITS_DB *db, ImageLink **imlink, int *Nsgroup)); -Group *find_image_tgroups PROTO((FITS_DB *db, ImageLink **imlink, int *Ntgroup)); -void fit_sgroup PROTO((Group *sgroup, int Nsgroup)); -void fit_tgroup PROTO((Group *tgroup, int Ntgroup)); -int gcatalog PROTO((Catalog *catalog)); -void initialize PROTO((int argc, char **argv)); -void initstats PROTO((char *mode)); -int liststats PROTO((double *value, double *dvalue, int N, StatType *stats)); -int load_images PROTO((FITS_DB *db)); -int main PROTO((int argc, char **argv)); -int open_graph PROTO((int N)); -void sort PROTO((unsigned int *X, int N)); -void sortB PROTO((double *X, double *Y, int N)); -void sortD PROTO((double *X, double *Y, double *Z, int N)); -void update_dvo_catalog PROTO((Catalog *catalog, Group *sgroup, int warn)); -void wcatalog PROTO((Catalog *catalog)); -void wimages PROTO((Image *image, int Nimage)); -void check_permissions PROTO((char *basefile)); -void lock_image_db PROTO((FITS_DB *db, char *filename)); -void unlock_image_db PROTO((FITS_DB *db)); -void create_image_db PROTO((FITS_DB *db)); -void set_db PROTO((FITS_DB *in)); -int Shutdown PROTO((char *format, ...) OHANA_FORMAT(printf, 1, 2) ); -void TrapSignal PROTO((int sig)); -void SetProtect PROTO((int mode)); -int SetSignals PROTO((void)); -int subset_images PROTO((FITS_DB *db)); -void update PROTO((FITS_DB *db, Group *sgroup, int Nsgroup)); -void sort_time PROTO((unsigned int *value, int N)); +void ConfigInit PROTO((int *argc, char **argv)); +void DonePlotting PROTO((Graphdata *graphmode, int N)); +void JpegPlot PROTO((Graphdata *graphmode, int N, char *filename)); +void PSPlot PROTO((Graphdata *graphmode, int N, char *filename)); +void PlotLabel PROTO((char *string, int N)); +void PlotVector PROTO((int Npts, double *vect, int mode, int N)); +void PrepPlotting PROTO((int Npts, Graphdata *graphmode, int N)); +void XDead PROTO((void)); +int args_uniphot PROTO((int argc, char **argv)); +int args_setphot PROTO((int argc, char **argv)); +void dumpresult PROTO((void)); +Group *find_image_sgroups PROTO((FITS_DB *db, ImageLink **imlink, int *Nsgroup)); +Group *find_image_tgroups PROTO((FITS_DB *db, ImageLink **imlink, int *Ntgroup)); +void fit_sgroup PROTO((Group *sgroup, int Nsgroup)); +void fit_tgroup PROTO((Group *tgroup, int Ntgroup)); +int gcatalog PROTO((Catalog *catalog)); +void initialize_uniphot PROTO((int argc, char **argv)); +void initialize_setphot PROTO((int argc, char **argv)); +void initstats PROTO((char *mode)); +int liststats PROTO((double *value, double *dvalue, int N, StatType *stats)); +int load_images_uniphot PROTO((FITS_DB *db)); +int main PROTO((int argc, char **argv)); +int open_graph PROTO((int N)); +void sort PROTO((unsigned int *X, int N)); +void sortB PROTO((double *X, double *Y, int N)); +void sortD PROTO((double *X, double *Y, double *Z, int N)); +void update_dvo_catalog PROTO((Catalog *catalog, Group *sgroup, int warn)); +void wcatalog PROTO((Catalog *catalog)); +void wimages PROTO((Image *image, int Nimage)); +void check_permissions PROTO((char *basefile)); +void lock_image_db PROTO((FITS_DB *db, char *filename)); +void unlock_image_db PROTO((FITS_DB *db)); +void create_image_db PROTO((FITS_DB *db)); +void set_db PROTO((FITS_DB *in)); +int Shutdown PROTO((char *format, ...) OHANA_FORMAT(printf, 1, 2) ); +void TrapSignal PROTO((int sig)); +void SetProtect PROTO((int mode)); +int SetSignals PROTO((void)); +int subset_images PROTO((FITS_DB *db)); +void update PROTO((FITS_DB *db, Group *sgroup, int Nsgroup)); +void sort_time PROTO((unsigned int *value, int N)); + +// setphot-specific prototypes +ZptTable *load_zpt_table PROTO((char *filename, int *nzpts)); +Image *load_images_setphot PROTO((FITS_DB *db, off_t *Nimage)); +int match_zpts_to_images PROTO((Image *image, off_t Nimage, ZptTable *zpts, int Nzpts)); +int update_setphot PROTO((Image *image, off_t Nimage)); +void update_catalog_setphot PROTO((Catalog *catalog, Image *image, off_t *index, off_t Nimage)); Index: /branches/pap/Ohana/src/uniphot/src/ConfigInit.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/ConfigInit.c (revision 28002) +++ /branches/pap/Ohana/src/uniphot/src/ConfigInit.c (revision 28003) @@ -1,3 +1,5 @@ # include "uniphot.h" + +void GetConfig (char *config, char *field, char *format, int N, void *ptr); void ConfigInit (int *argc, char **argv) { @@ -18,5 +20,5 @@ if (VERBOSE) fprintf (stderr, "loaded config file: %s\n", file); - ScanConfig (config, "CATDIR", "%s", 0, CATDIR); + GetConfig (config, "CATDIR", "%s", 0, CATDIR); ScanConfig (config, "CATMODE", "%s", 0, CATMODE); ScanConfig (config, "CATFORMAT", "%s", 0, CATFORMAT); @@ -24,4 +26,12 @@ sprintf (ImageCat, "%s/Images.dat", CATDIR); + + GetConfig (config, "GSCFILE", "%s", 0, GSCFILE); + if (!ScanConfig (config, "SKY_DEPTH", "%d", 0, &SKY_DEPTH)) { + SKY_DEPTH = 2; + } + if (!ScanConfig (config, "SKY_TABLE", "%s", 0, SKY_TABLE)) { + SKY_TABLE[0] = 0; + } ScanConfig (config, "ZERO_PT", "%lf", 0, &ZERO_POINT); @@ -42,2 +52,14 @@ } + +void GetConfig (char *config, char *field, char *format, int N, void *ptr) { + + char *status; + + status = ScanConfig (config, field, format, N, ptr); + if (status == NULL) { + fprintf (stderr, "error in config, cannot find %s\n", field); + exit (1); + } + return; +} Index: /branches/pap/Ohana/src/uniphot/src/Shutdown.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/Shutdown.c (revision 28002) +++ /branches/pap/Ohana/src/uniphot/src/Shutdown.c (revision 28003) @@ -23,5 +23,5 @@ SetProtect (TRUE); gfits_db_close (db); - fprintf (stderr, "ERROR: addstar halted\n"); + fprintf (stderr, "ERROR: halted\n"); exit (1); } Index: /branches/pap/Ohana/src/uniphot/src/args.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/args.c (revision 28002) +++ /branches/pap/Ohana/src/uniphot/src/args.c (revision 28003) @@ -1,5 +1,5 @@ # include "uniphot.h" -int args (int argc, char **argv) { +int args_uniphot (int argc, char **argv) { int N; @@ -72,2 +72,26 @@ } +int args_setphot (int argc, char **argv) { + + int N; + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + UPDATE = FALSE; + if ((N = get_argument (argc, argv, "-update"))) { + remove_argument (N, &argc, argv); + UPDATE = TRUE; + } + + if (argc != 2) { + fprintf (stderr, "ERROR: USAGE: setphot (zptfile) [options]\n"); + exit (2); + } + + return (TRUE); +} + Index: /branches/pap/Ohana/src/uniphot/src/initialize.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/initialize.c (revision 28002) +++ /branches/pap/Ohana/src/uniphot/src/initialize.c (revision 28003) @@ -1,9 +1,9 @@ # include "uniphot.h" -void initialize (int argc, char **argv) { +void initialize_uniphot (int argc, char **argv) { /* are these set correctly? */ ConfigInit (&argc, argv); - args (argc, argv); + args_uniphot (argc, argv); if ((photcode = GetPhotcodebyName (argv[1])) == NULL) { @@ -21,2 +21,9 @@ } +void initialize_setphot (int argc, char **argv) { + + /* are these set correctly? */ + ConfigInit (&argc, argv); + args_setphot (argc, argv); +} + Index: /branches/pap/Ohana/src/uniphot/src/load_images.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/load_images.c (revision 28002) +++ /branches/pap/Ohana/src/uniphot/src/load_images.c (revision 28003) @@ -1,5 +1,5 @@ # include "uniphot.h" -int load_images (FITS_DB *db) { +int load_images_uniphot (FITS_DB *db) { if (VERBOSE) fprintf (stderr, "finding images\n"); @@ -10,2 +10,19 @@ return (TRUE); } + +Image *load_images_setphot (FITS_DB *db, off_t *Nimage) { + + Image *image; + + if (VERBOSE) fprintf (stderr, "finding images\n"); + + /* read entire db table */ + if (!dvo_image_load (db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db[0].filename); + + /* use a vtable to keep the images to be calibrated */ + image = gfits_table_get_Image (&db[0].ftable, Nimage, &db[0].swapped); + + fprintf (stderr, "loaded %lld images\n", (long long) *Nimage); + + return (image); +} Index: /branches/pap/Ohana/src/uniphot/src/load_zpt_table.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/load_zpt_table.c (revision 28003) +++ /branches/pap/Ohana/src/uniphot/src/load_zpt_table.c (revision 28003) @@ -0,0 +1,41 @@ +# include "uniphot.h" + +ZptTable *load_zpt_table (char *filename, int *nzpts) { + + int Nzpts, NZPTS; + ZptTable *zpts; + double zpt, mjd; + + FILE *f; + + f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open zpt table file %s\n", filename); + exit (1); + } + + Nzpts = 0; + NZPTS = 100; + ALLOCATE (zpts, ZptTable, NZPTS); + + // format is fixed: (time) (zpt) + int status; + while ((status = fscanf (f, "%lf %lf", &mjd, &zpt)) == 2) { + zpts[Nzpts].zpt = zpt; + zpts[Nzpts].time = ohana_mjd_to_sec (mjd); + zpts[Nzpts].found = FALSE; + + Nzpts ++; + CHECK_REALLOCATE (zpts, ZptTable, NZPTS, Nzpts, 100); + } + + if (status != EOF) { + fprintf (stderr, "unexpected formatting on line %d\n", Nzpts); + exit (2); + } + + fprintf (stderr, "loaded %d zero points\n", Nzpts); + + *nzpts = Nzpts; + return zpts; +} Index: /branches/pap/Ohana/src/uniphot/src/match_zpts_to_images.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/match_zpts_to_images.c (revision 28003) +++ /branches/pap/Ohana/src/uniphot/src/match_zpts_to_images.c (revision 28003) @@ -0,0 +1,90 @@ +# include "uniphot.h" + +/* sort a coordinate pair (X,Y) and the associated index (S) */ +static void sort_zpts_by_time (ZptTable *zpts, int Nzpts) { + +# define SWAPFUNC(A,B){ ZptTable tmp; \ + tmp = zpts[A]; zpts[A] = zpts[B]; zpts[B] = tmp; \ + } +# define COMPARE(A,B)(zpts[A].time < zpts[B].time) + + OHANA_SORT (Nzpts, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE +} + +# define SCALE 0.001 + +int match_zpts_to_images (Image *image, off_t Nimage, ZptTable *zpts, int Nzpts) { + + // I have two lists. I need to match images->tzero to zpts->time. multiple images may match a single zpt + + // sort both lists by time (or at least get sorted indices) + // sweep through both lists, advancing the one that is lagging + // apply the matches + + int dT, NImatch, Nmatch; + off_t i, Ni, Nz, *index; + PhotCode *code; + + // create index and sort + ALLOCATE (index, off_t, Nimage); + for (i = 0; i < Nimage; i++) { + index[i] = i; + } + sort_image_subset(image, index, Nimage); // slightly misnamed sort function from libdvo + + // sort the zpts + sort_zpts_by_time (zpts, Nzpts); + + NImatch = 0; // matched images + for (i = Nz = 0; (i < Nimage) && (Nz < Nzpts); ) { + + if (i % 1000 == 0) fprintf (stderr, "."); + if (Nz % 100 == 0) fprintf (stderr, "!"); + + Ni = index[i]; + dT = image[Ni].tzero - zpts[Nz].time; + + // negative dT, i is too small (allow a 1sec overlap window) + if (dT < -1) { + i++; + continue; + } + + // XXX careful about definition of image->tzero and zpt->time + // negative dT, i is too small (allow a 1sec overlap window) + if (dT > +1) { + Nz++; + continue; + } + + // check that we have a valid photcode (skip mosaic images) + code = GetPhotcodebyCode(image[Ni].photcode); + if (!code) { + i++; + continue; + } + + // we have a match: set zpt and record the match + image[Ni].Mcal = SCALE*code[0].C - zpts[Nz].zpt; + image[Ni].flags &= ~ID_IMAGE_NOCAL; // clear the NOCAL flag + zpts[Nz].found = TRUE; + NImatch ++; + + // advance the image counter only -- a single zpt may match more than one image + i++; + } + + // how many zpts have we matched? + Nmatch = 0; + for (Nz = 0; Nz < Nzpts; Nz++) { + if (zpts[Nz].found) Nmatch ++; + } + + fprintf (stderr, "found %d zpt matches, %d image matches\n", Nmatch, NImatch); + + return (TRUE); +} + Index: /branches/pap/Ohana/src/uniphot/src/setphot.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/setphot.c (revision 28003) +++ /branches/pap/Ohana/src/uniphot/src/setphot.c (revision 28003) @@ -0,0 +1,49 @@ +# include "uniphot.h" + +int main (int argc, char **argv) { + + off_t Nimage; + int status, Nzpts; + FITS_DB db; + ZptTable *zpts; + Image *image; + + /* get configuration info, args, lockfile */ + initialize_setphot (argc, argv); + + set_db (&db); + status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT)); + if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + if (db.dbstate == LCK_EMPTY) Shutdown ("ERROR: No images in catalog %s (1)", db.filename); + + zpts = load_zpt_table (argv[1], &Nzpts); + + // load images + image = load_images_setphot (&db, &Nimage); + if (!UPDATE) dvo_image_unlock (&db); + + match_zpts_to_images (image, Nimage, zpts, Nzpts); + + update_setphot(image, Nimage); + + // write image table + if (UPDATE) { + dvo_image_save (&db, VERBOSE); + } + + exit (0); +} + + +/* setphot : set the zero points for images in the db (perhaps based on external information) + setphot (zpt_table) + + * load text table of zpts, time (photcode?) + * load images + * match images to zpts + * set zpts + * load catalogs + * update detection (& averages?) + + */ + Index: /branches/pap/Ohana/src/uniphot/src/uniphot.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/uniphot.c (revision 28002) +++ /branches/pap/Ohana/src/uniphot/src/uniphot.c (revision 28003) @@ -9,5 +9,5 @@ /* get configuration info, args, lockfile */ - initialize (argc, argv); + initialize_uniphot (argc, argv); set_db (&db); @@ -18,5 +18,5 @@ /* load images */ - load_images (&db); + load_images_uniphot (&db); /* filter image list by selection */ Index: /branches/pap/Ohana/src/uniphot/src/update_catalog_setphot.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/update_catalog_setphot.c (revision 28003) +++ /branches/pap/Ohana/src/uniphot/src/update_catalog_setphot.c (revision 28003) @@ -0,0 +1,97 @@ +# include "uniphot.h" +# include + +int update_setphot (Image *image, off_t Nimage) { + + SkyRegion UserPatch; + SkyTable *sky = NULL; + SkyList *skylist = NULL; + Catalog catalog; + off_t i, maxID, *index; + + // create an index for the image IDs + maxID = 0; + for (i = 0; i < Nimage; i++) { + maxID = MAX(maxID, image[i].imageID); + } + ALLOCATE (index, off_t, maxID + 1); + for (i = 0; i < Nimage; i++) { + if (!image[i].imageID) continue; // images with ID == 0 are virtual + index[image[i].imageID] = i; + } + + + // load the current sky table (layout of all SkyRegions) + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // determine the populated SkyRegions overlapping the requested area (default depth) + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + skylist = SkyListByPatch (sky, -1, &UserPatch); + + // update measurements for each populated catalog + for (i = 0; i < skylist[0].Nregions; i++) { + + // set up the basic catalog info + catalog.filename = skylist[0].filename[i]; + catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naves_disk) { + if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + update_catalog_setphot (&catalog, image, index, Nimage); + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename); + + dvo_catalog_save (&catalog, VERBOSE); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + } + return (TRUE); +} + +void update_catalog_setphot (Catalog *catalog, Image *image, off_t *index, off_t Nimage) { + + off_t i, j, m, id, idx, found; + float Mcal; + + found = 0; + for (i = 0; i < catalog[0].Naverage; i++) { + + m = catalog[0].average[i].measureOffset; + for (j = 0; j < catalog[0].average[i].Nmeasure; j++, m++) { + idx = catalog[0].measure[m].imageID; + if (idx <= 0) continue; // detections with imageID == 0 do not have a valid image (eg, ref photcode) + + id = index[idx]; + Mcal = image[id].Mcal; + + catalog[0].measure[m].Mcal = image[id].Mcal; + found ++; + } + } + + if (found) { + fprintf (stderr, "found %lld matches\n", (long long) found); + } +} Index: /branches/pap/PS-IPP-Metadata-Config/lib/PS/IPP/Metadata/Config.pm =================================================================== --- /branches/pap/PS-IPP-Metadata-Config/lib/PS/IPP/Metadata/Config.pm (revision 28002) +++ /branches/pap/PS-IPP-Metadata-Config/lib/PS/IPP/Metadata/Config.pm (revision 28003) @@ -59,4 +59,36 @@ } +# Parse a list of METADATAs +sub parse_list +{ + my $self = shift; # This parser + + # Split input into separate METADATAs because the parser seems to take forever handling those together + my @whole = split /\n/, join( '', @_ ); # The whole input + my @single; # A single METADATA element + my @output; # Output + foreach my $value (@whole) { + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $md = join("\n", @single); # Metadata to parse + my $parsed = $self->parse($md) or (carp "Unable to parse input: $md\n" and next); + (carp "More than one entry in input: $md\n" and next) if scalar @$parsed > 1; + my $metadata = shift @$parsed; # Metadata of interest + (carp "MD element is not of type METADATA: $md\n" and next) if $metadata->{class} ne "metadata"; + my $components = $metadata->{value}; # Components of metadata + my %hash; # Hash for entry + foreach my $comp (@$components) { + $hash{$comp->{name}} = $comp->{value}; + } + push @output, \%hash; + @single = (); + } + } + + return \@output; +} + sub _merge_duplicates { my ( $self, $tree ) = @_; @@ -73,5 +105,5 @@ # stop if the prevous pass removed the last element and called redo last unless defined $elem; - + # recurse through nested metadata if ( $elem->{class} eq "metadata" ) { @@ -83,5 +115,5 @@ delete $elem->{multi}; next; - } + } if ( defined $names{ $elem->{name} } ) { Index: /branches/pap/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /branches/pap/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 28002) +++ /branches/pap/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 28003) @@ -17,4 +17,5 @@ resolve_project getCamRunByCamID + parse_md_fast ); our %EXPORT_TAGS = (standard => [@EXPORT_OK]); @@ -27,4 +28,11 @@ use PS::IPP::Config qw( :standard ); use Carp; +use File::Temp qw(tempfile); +use File::Basename; +use POSIX; +use Time::HiRes qw(gettimeofday); + +my $dvo_verbose = 0; +my $save_temps = 0; # caches of camProcessedExp objects. @@ -34,10 +42,14 @@ my %camRunByCamIDCache; -### my @images = locate_images($image_db, $req_type, $img_type, $id, $component, -### $mjd_min, $mjd_max, $filter); +# cache of last project looked up +my $last_project = ""; + +my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool); +my ($pstamptool, $dvoImagesAtCoords, $whichimage, $ppConfigDump); sub locate_images { my $ipprc = shift; # required - my $image_db = shift; # required + my $imagedb = shift; # required + my $rowList = shift; # required my $req_type = shift; # required my $img_type = shift; # required @@ -47,6 +59,4 @@ my $option_mask = shift; my $need_magic = shift; - my $x = shift; - my $y = shift; my $mjd_min = shift; my $mjd_max = shift; @@ -56,6 +66,7 @@ # we die in response to bad data in request files - # The wrapper script is responsible for updating the database - die "Unknown req_type: $req_type" + # The caller is responsible for updating the database + # pstampparse.pl error checks now so this shouldn't happen + &my_die("Unknown req_type: $req_type", $PS_EXIT_PROG_ERROR) if ($req_type ne "byid") and ($req_type ne "byexp") and @@ -84,5 +95,10 @@ if ($req_type eq "bycoord") { - my $results = lookup_bycoord($ipprc, $image_db, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); + my $num_rows = scalar @$rowList; + die "Unexpected number of rows found in rowList: $num_rows" if $num_rows != 1; + my $row = $rowList->[0]; + my $x = $row->{CENTER_X}; + my $y = $row->{CENTER_Y}; + my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); return $results; } @@ -93,10 +109,12 @@ # in one place $req_type = "bydiff"; - my $results = lookup_diff($ipprc, $image_db, $id, $component, 1, $option_mask, $img_type, $verbose); + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $img_type, $verbose); return $results; } if ($req_type eq "bydiff") { - my $results = lookup_diff($ipprc, $image_db, $id, $component, 0, $option_mask, $img_type, $verbose); + # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or + # the image from the diffRun + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $img_type, $verbose); if (!$results) { return undef; @@ -130,5 +148,5 @@ # fall though and lookup by stack_id } else { - # shouldn't I check this elsewhere? + # This is checked this elsewhere? print STDERR "Error: $img_type is an unknown image type\n"; return undef; @@ -145,16 +163,19 @@ } - my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic, - $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); + my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component, + $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); return $results; } -sub lookup -{ +# The subroutine lookup handles byexp, byid, and byskycell lookups including lookups that are +# triggered by a bydiff or bycoord request + +sub lookup { my $ipprc = shift; - my $image_db = shift; + my $rowList = shift; + my $imagedb = shift; my $req_type = shift; - my $img_type = shift; + my $stage = shift; my $id = shift; my $tess_id = shift; @@ -172,14 +193,4 @@ my $use_imfile_id = ($req_type eq "byid") && ($option_mask & $PSTAMP_USE_IMFILE_ID); - my $missing_tools; - my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); - my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); - my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); - my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); - my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } my $command; my $id_opt; # option for the lookup @@ -195,30 +206,49 @@ my $class_id; my $skycell_id; - - if (isnull($component)) { - $component = undef; - } + my $default_error = $PSTAMP_NO_IMAGE_MATCH; # note $magic_arg may be cleared below depending on $req_type my $magic_arg = $need_magic ? " -destreaked" : ""; + # all rows have the same center type + my $skycenter = ! ($rowList->[0]->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); + my $component_args; - if ($img_type eq "raw") { - $class_id = $component; - $command = "$regtool -processedimfile -dbname $image_db"; + my $choose_components = 0; + if ($stage eq "raw") { + $command = "$regtool -processedimfile -dbname $imagedb"; # XXX: for now restrict lookups to type object # are stamps of detrend exposures interesting? $command .= " -exp_type object"; $id_opt = $use_imfile_id ? "-raw_imfile_id" : "-exp_id"; - $component_args = " -class_id $class_id" if $component; + + if ($component and $component ne 'all') { + $class_id = $component; + $component_args = " -class_id $class_id"; + } $want_astrom = 1; $set_class_id = 1; - } elsif ($img_type eq "chip") { - $class_id = $component; - $command = "$chiptool -processedimfile -dbname $image_db"; - $component_args = " -class_id $class_id" if $class_id; + } elsif ($stage eq "chip") { + # if the request is such that it will yield a single image per "run" or the + # center is specified in pixel coordinates + # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then + # choose the chips containing the center pixel by calling selectComponets() below + if ($component or $use_imfile_id or !$skycenter) { + $command = "$chiptool -processedimfile -dbname $imagedb"; + if ($component and $component ne 'all') { + $class_id = $component; + $component_args = " -class_id $class_id"; + } + } else { + $command = "$chiptool -listrun -pstamp_order -dbname $imagedb"; + $choose_components = 1; + } $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id"; + # XXX: perhaps we should stop resolving images to this level here and do it at job run time. + # With the mdc file it has all of the information that it needs. $image_name = "PPIMAGE.CHIP"; - $mask_name = "PPIMAGE.CHIP.MASK"; + # $mask_name = "PPIMAGE.CHIP.MASK"; + # XXX: really need to handle this properly! + $mask_name = "PSASTRO.OUTPUT.MASK"; $weight_name = "PPIMAGE.CHIP.VARIANCE"; $cmf_name = "PSPHOT.OUTPUT"; @@ -228,8 +258,15 @@ $want_astrom = 1; $set_class_id = 1; - } elsif ($img_type eq "warp") { - $skycell_id = $component; - $command = "$warptool -warped -dbname $image_db"; - $component_args = " -skycell_id $skycell_id" if $skycell_id; + } elsif ($stage eq "warp") { + if ($component or $use_imfile_id or !$skycenter) { + $command = "$warptool -warped -dbname $imagedb"; + if ($component and $component ne 'all') { + $skycell_id = $component; + $component_args = " -skycell_id $skycell_id"; + } + } else { + $command = "$warptool -listrun -pstamp_order -dbname $imagedb"; + $choose_components = 1; + } $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id"; $image_name = "PSWARP.OUTPUT"; @@ -239,8 +276,15 @@ $psf_name = "PSPHOT.PSF.SKY.SAVE"; $base_name = "path_base"; # name of the field for the warptool output - } elsif ($img_type eq "diff") { - $skycell_id = $component; - $command = "$difftool -diffskyfile -dbname $image_db"; - $component_args = " -skycell_id $skycell_id" if $skycell_id; + } elsif ($stage eq "diff") { + if ($component or $use_imfile_id or !$skycenter) { + $command = "$difftool -diffskyfile -dbname $imagedb"; + if ($component and $component ne 'all') { + $skycell_id = $component; + $component_args = " -skycell_id $skycell_id"; + } + } else { + $command = "$difftool -listrun -pstamp_order -dbname $imagedb"; + $choose_components = 1; + } $id_opt = $use_imfile_id ? "-diff_skyfile_id" : "-diff_id"; $image_name = "PPSUB.OUTPUT"; @@ -248,9 +292,9 @@ $weight_name = "PPSUB.OUTPUT.VARIANCE"; $cmf_name = "PPSUB.OUTPUT.SOURCES"; - $psf_name = "PSPHOT.PSF.SKY.SAVE"; + $psf_name = "PSPHOT.PSF.SKY.SAVE"; $base_name = "path_base"; - } elsif ($img_type eq "stack") { + } elsif ($stage eq "stack") { $skycell_id = $component; - $command = "$stacktool -sumskyfile -dbname $image_db"; + $command = "$stacktool -sumskyfile -dbname $imagedb"; $id_opt = "-stack_id"; $component_args = " -skycell_id $skycell_id" if $skycell_id; @@ -272,7 +316,8 @@ $base_name = "path_base"; } else { - die "Unknown img_type supplied: $img_type"; - } - + die "Unknown IMG_TYPe supplied: $stage"; + } + + my $filter_runs = 0; if ($req_type eq "byid") { $command .= " $id_opt $id"; @@ -288,12 +333,15 @@ # the reason as 'not destreaked' $magic_arg = ""; + # remove duplicate runs for the same exposure. + $filter_runs = 1; } elsif ($req_type eq "byskycell") { die "tess_id and component are required for byskycell" if !$tess_id or ! $skycell_id; $command .= " -tess_id $tess_id -skycell_id $skycell_id"; } else { - die "Unknown req_type supplied: $req_type"; - } - - if ($img_type ne "stack") { + # this should be caught by caller + &my_die("Unexpected req_type supplied: $req_type", $PS_EXIT_PROG_ERROR); + } + + if ($stage ne "stack") { $command .= $magic_arg; $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin; @@ -304,30 +352,32 @@ $command .= " -data_group $data_group" if !isnull($data_group); - # run the tool and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - # not sure if we should die here - print STDERR @$stderr_buf; + my $images = runToolAndParse($command, $verbose); + + if (!$images or scalar @$images == 0) { return undef; } - my $buf = join "", @$stdout_buf; - if (!$buf) { - return; - } - - my $mdcParser = PS::IPP::Metadata::Config->new; - my $images = parse_md_fast($mdcParser, $buf) - or die ("Unable to parse metadata config doc"); - + if ($filter_runs) { + # The image selectors are such that multiple runs my have be returned for the same exposure. + # Return only the latest one. + $images = filterRuns($stage, $need_magic, $images, $verbose); + } + if ($choose_components) { + # the list of "images" is actually a list of "Runs" + # match the coords in the rows to the components in the runs + # returns an actual list of images + $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose); + } else { + # put index for each of the rows into all of the the images + setRowRefs($rowList, $images); + } + + my $camera; my $output = []; - - my $camera; foreach my $image (@$images) { my $base; next if $image->{fault}; - next if ($img_type ne "raw") and $image->{quality}; + next if ($stage ne "raw") and $image->{quality}; if ($base_name) { @@ -348,26 +398,27 @@ $ipprc->define_camera($camera); } + # ok it's a keeper my $out = $image; # if uri is nil this will get overridded below # (we do this here for raw stage) - $out->{image} = $image->{uri}; + $out->{image} = $out->{uri}; if ($set_class_id) { - $class_id = $image->{class_id}; + $class_id = $out->{class_id}; $out->{component} = $class_id; } else { - $out->{component} = $image->{skycell_id}; + $out->{component} = $out->{skycell_id}; } my $stage_id; - if ($img_type eq "raw") { - $stage_id = $image->{exp_id}; - } elsif ($img_type eq "chip") { - $stage_id = $image->{chip_id}; - } elsif ($img_type eq "warp") { - $stage_id = $image->{warp_id}; - } elsif ($img_type eq "diff") { - $stage_id = $image->{diff_id}; - if ($inverse && $image->{bothways}) { + if ($stage eq "raw") { + $stage_id = $out->{exp_id}; + } elsif ($stage eq "chip") { + $stage_id = $out->{chip_id}; + } elsif ($stage eq "warp") { + $stage_id = $out->{warp_id}; + } elsif ($stage eq "diff") { + $stage_id = $out->{diff_id}; + if ($inverse && $out->{bothways}) { $image_name = "PPSUB.INVERSE"; $mask_name = "PPSUB.INVERSE.MASK"; @@ -375,20 +426,33 @@ $cmf_name = "PPSUB.INVERSE.SOURCES"; } - } elsif ($img_type eq "stack") { - $stage_id = $image->{stack_id}; + } elsif ($stage eq "stack") { + $stage_id = $out->{stack_id}; } $out->{stage_id} = $stage_id; - $out->{stage} = $img_type; - $out->{image_db} = $image_db; + $out->{stage} = $stage; + $out->{imagedb} = $imagedb; + + my $mask_base; + if ($want_astrom) { + if (! defined $out->{astrom}) { + if (! find_astrometry($ipprc, $imagedb, $out, $verbose)) { + print STDERR "failed to find astrometry for $stage $stage_id\n"; + next; + } + } + # XXX: do this right by looking at the recipe + $mask_base = $out->{cam_path_base}; + } else { + $mask_base = $base; + } if ($base) { $out->{image} = $ipprc->filename($image_name, $base, $class_id) if $image_name; - $out->{mask} = $ipprc->filename($mask_name, $base, $class_id) if $mask_name; + $out->{mask} = $ipprc->filename($mask_name, $mask_base, $class_id) if $mask_name; $out->{weight} = $ipprc->filename($weight_name, $base, $class_id) if $weight_name; - $out->{cmf} = $ipprc->filename($cmf_name, $base, $class_id) if $cmf_name; - $out->{psf} = $ipprc->filename($psf_name, $base, $class_id) if $psf_name; - $out->{backmdl}= $ipprc->filename($backmdl_name, $base, $class_id) if $backmdl_name; - } - $out->{astrom} = find_astrometry($ipprc, $image_db, $image, $verbose) if $want_astrom; + $out->{cmf} = $ipprc->filename($cmf_name, $base, $class_id) if $cmf_name; + $out->{psf} = $ipprc->filename($psf_name, $base, $class_id) if $psf_name; + $out->{backmdl}= $ipprc->filename($backmdl_name,$base, $class_id) if $backmdl_name; + } push @$output, $out; @@ -397,7 +461,9 @@ return $output; } + sub lookup_diff { my $ipprc = shift; - my $image_db = shift; + my $rowList = shift; + my $imagedb = shift; my $id = shift; my $skycell_id = shift; @@ -409,43 +475,32 @@ my $inverse = $option_mask & $PSTAMP_SELECT_INVERSE; - my $missing_tools; - my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } - - my $command = "$difftool -diffskyfile -dbname $image_db"; + my $command = "$difftool -dbname $imagedb"; + my $listrun = 0; if ($byid) { - $command .= " -diff_id $id"; + if ($skycell_id) { + $command .= " -diffskyfile -diff_id $id -skycell_id $skycell_id"; + } else { + $command .= " -listrun -diff_id $id"; + $listrun = 1; + } } else { - $command .= " -diff_skyfile_id $id"; - } - $command .= " -skycell_id $skycell_id" if $skycell_id; - - # run the tool and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - # not sure if we should die here - print STDERR @$stderr_buf; - return undef; - } - - my $buf = join "", @$stdout_buf; - if (!$buf) { - return undef; - } - - my $mdcParser = PS::IPP::Metadata::Config->new; - my $images = parse_md_fast($mdcParser, $buf) - or die ("Unable to parse metadata config doc"); - - my $n = @$images; - if (!$byid && ($n > 1)) { + $command .= " -diffskyfile -diff_skyfile_id $id"; + } + + my $output = runToolAndParse($command, $verbose); + + my $n = $output ? scalar @$output : 0; + if (!$listrun && ($n > 1)) { die ("difftool returned an unexpected number of diffskyfiles: $n"); } elsif ($n == 0) { return undef; + } + + my $images; + if ($listrun) { + $images = selectComponents($ipprc, $imagedb, 'byid', 'diff', $rowList, $output, $verbose); + } else { + $images = $output; } @@ -478,5 +533,7 @@ if (($img_type ne "diff") and ($img_type ne "stack")) { print STDERR "lookup_diff: cannot lookup IMG_TYPE $img_type bydiff from a stack stack diff run\n"; - next; + setErrorCodes($rowList, $PSTAMP_INVALID_REQUEST); + # all images will be the same so we can stop + last; } # stack-stack diff @@ -527,4 +584,7 @@ if ($img_type eq "diff") { + my @imageList = ($image); + + setRowRefs($rowList, \@imageList); # the $image is going to be returned directly in this case so we need to duplicate # some of processing that lookup does for other img_types @@ -539,9 +599,9 @@ $image->{stage_id} = $image->{diff_id}; $image->{stage} = "diff"; - $image->{image_db} = $image_db; + $image->{imagedb} = $imagedb; $image->{component} = $image->{skycell_id}; } else { # XXX this will only happen if the minuend is not a warp. See hack above - print STDERR "WARNING: cannot resolve camera so cannot get resolve file rules\n"; + print STDERR "WARNING: cannot resolve camera so cannot find file rules\n"; next; } @@ -554,5 +614,6 @@ sub lookup_bycoord { my $ipprc = shift; - my $image_db = shift; + my $rowList = shift; + my $imagedb = shift; my $img_type = shift; my $tess_id = shift; @@ -568,23 +629,15 @@ my $verbose = shift; - my $missing_tools; - my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); - my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); - my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1); - my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } - my $results = (); if (($img_type eq "raw") or ($img_type eq "chip")) { - my $runs = lookup_runs_by_camid_and_coords($ipprc, $image_db, $img_type, + my $runs = lookup_runs_by_cam_id_and_coords($ipprc, $imagedb, $img_type, $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose); + # lookup is going to filter these we don't need to do it here +# $runs = filterRuns($img_type, $need_magic, $runs, $verbose); foreach my $run (@$runs) { next if $component and ($run->{component} ne $component); - my $these_results = lookup($ipprc, $image_db, "byid", $img_type, $run->{id}, + my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $run->{id}, $tess_id, $run->{component}, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); @@ -602,5 +655,5 @@ foreach my $skycell (@$skycells) { - my $these_results = lookup($ipprc, $image_db, "byskycell", $img_type, undef, + my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef, $skycell->{tess_id}, $skycell->{component}, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); @@ -614,10 +667,10 @@ } -# lookup_runs_by_camid_and_coords +# lookup_runs_by_cam_id_and_coords # given an ra, dec, and optionally other paramters, find camera runs for exposures # that are within some distance of the coordinates -sub lookup_runs_by_camid_and_coords { +sub lookup_runs_by_cam_id_and_coords { my $ipprc = shift; - my $image_db = shift; + my $imagedb = shift; my $img_type = shift; my $ra = shift; @@ -631,14 +684,7 @@ my $camruns; - my $missing_tools; - my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); - my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } { - my $command = "$camtool -dbname $image_db -processedexp"; + my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order"; $command .= " -ra $ra -decl $dec -radius 1.6"; $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin; @@ -652,19 +698,7 @@ $command .= " -data_group $data_group" if $data_group; $command .= " -destreaked" if $need_magic; + # run the tool and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; - return undef; - } - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - - my $output = join "", @$stdout_buf; - if (!$output) { - print STDERR "no output returned from $command\n" if $verbose; - return undef; - } - $camruns = parse_md_fast($mdcParser, $output); + $camruns = runToolAndParse($command, $verbose); } if (!$camruns) { @@ -672,11 +706,18 @@ } my $runs; + my $last_exp_id = 0; foreach my $camRun (@$camruns) { my $cam_id = $camRun->{cam_id}; - - updateCamRunCache($camRun); + my $exp_id = $camRun->{exp_id}; + + next if $exp_id eq $last_exp_id; next if $camRun->{quality}; next if $camRun->{fault}; + + updateCamRunCache($camRun); + + $last_exp_id = $exp_id; + # XXX Use file rule my $astrom = $camRun->{path_base} . ".smf"; @@ -684,12 +725,15 @@ next if !$astrom_resolved; + my $start_dvo = gettimeofday(); my $command = "$dvoImagesAtCoords -astrom $astrom_resolved $ra $dec"; # run the tool and parse the output my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + run(command => $command, verbose => $dvo_verbose); + my $dtime_dvo = gettimeofday() - $start_dvo; +# print "Time to run dvoImagesAtCoords: $dtime_dvo\n"; unless ($success) { my $result_code = $error_code >> 8; if ($result_code == $PSTAMP_NO_OVERLAP) { - print STDERR "no overlap for $astrom\n" if $verbose; + print "no overlap for $astrom\n" if $verbose; next; } @@ -709,10 +753,14 @@ my $n = scalar @lines; if ($n != 1) { - print STDERR "unexpected number of lines returned by dvoImagesAtCoords: $n\n"; - # XXX: There is a bug in dvo where each component is listed twice # When that gets fixed remove the conditional and just die - die "unexpected number lines returned by dvoImagesAtCoords: $n" - if ($n != 2) or ($lines[0] ne $lines[1]); + if ($n ne 2 or $lines[0] ne $lines[1]) { + print STDERR "unexpected number of lines returned by dvoImagesAtCoords: $n\n"; + print STDERR "OUTPUT:\n$output\n"; + # actually this seems to happen sometimes. Probably due to a problem with + # astrometry. Just skip this camRun + print STDERR "skipping: $astrom\n"; + next; + } } @@ -721,4 +769,7 @@ die "unexpected output from dvoImagesAtCoords: $lines[0]"; } + + print "cam_id $cam_id exp_id $camRun->{exp_id} chip_id $camRun->{chip_id} $class_id\n"; + # build the hash to return my $run = { @@ -727,12 +778,16 @@ chip_id => $camRun->{chip_id}, cam_id => $camRun->{cam_id}, - astrom => $astrom_resolved, + astrom => $astrom_resolved, # XXX: is astrom used? class_id => $class_id, component => $class_id }; if ($img_type eq "chip") { - $run->{id} = $run->{chip_id}; + $run->{id} = $camRun->{chip_id}; + $run->{state} = $camRun->{chip_state}; + $run->{magicked} = $camRun->{chip_magicked}; } else { - $run->{id} = $run->{exp_id}; + $run->{id} = $camRun->{exp_id}; + $run->{state} = 'full'; + $run->{magicked} = $camRun->{raw_magicked}; } push @$runs, $run; @@ -749,11 +804,4 @@ my $dec = shift; my $verbose = shift; - - my $missing_tools; - my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } $requested_tess_id = "" if isnull($requested_tess_id); @@ -810,12 +858,13 @@ # cache of results of ppConfigDump my %astromSources; -my $last_exp_id = 0; -my $lastAstromFile; # find the astrometry file for a given exposure # return undef if no completed camRun exists +# XXX Right now this probably only works cameras where the astrometry is done at the camera stage +# What other possibilities are there for ASTROM.SOURCE besides PSASTRO.OUTPUT? + sub find_astrometry { my $ipprc = shift; - my $image_db = shift; + my $imagedb = shift; my $image = shift; # hashref to output of the lookup tool my $verbose = shift; @@ -824,22 +873,8 @@ my $exp_id = $image->{exp_id}; - if (($exp_id eq $last_exp_id) and $lastAstromFile) { - # running camtool 60 times is really expensive when the answer is the same - return $lastAstromFile; - } - $last_exp_id = 0; - $lastAstromFile = undef; - - my $missing_tools; - my $camtool = can_run("camtool") or (warn "Can't find camtool" and $missing_tools = 1); - my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } my $camRun = getCamRunByExpID($exp_id); if (!$camRun) { - my $command = "$camtool -dbname $image_db -processedexp -exp_id $exp_id"; + my $command = "$camtool -dbname $imagedb -processedexp -exp_id $exp_id -pstamp_order"; # run the tool and parse the output my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -847,5 +882,5 @@ unless ($success) { print STDERR @$stderr_buf; - return undef; + return 0; } @@ -855,20 +890,23 @@ if (!$output) { print STDERR "no output returned from $command\n" if $verbose; - return undef; + return 0; } my $camruns = parse_md_fast($mdcParser, $output); if (!$camruns) { - return undef; + return 0; } # If there are multiple cam runs for this exposure, take the last completed one with good quality # on the assumption that it has the best astrometry. - while ($camRun = pop @$camruns) { - last if (($camRun->{quality} eq 0) and ($camRun->{fault} eq 0)); + foreach my $cr (@$camruns) { + if (($cr->{state} eq 'full') and ($cr->{quality} eq 0) and ($cr->{fault} eq 0)) { + $camRun = $cr; + last; + } } # XXX: this looks like a bug at least if ASTROM.SOURCE eq PSASTRO.OUTPUT if (!$camRun) { # no cam runs for this exposure id therefore best astrometry is whatever is in the header - return undef; + return 0; } updateCamRunCache($camRun); @@ -899,10 +937,13 @@ # XXX: Is this code correct if ASTROM.SOURCE ne "PSASTRO.OUTPUT" my $astromFile = $ipprc->filename($astromSource, $camRoot); - if ($astromFile) { - $lastAstromFile = $astromFile; - $last_exp_id = $exp_id; - } - - return $astromFile; + if (!$astromFile) { + print STDERR "failed to find astrometry file from $astromSource $camRoot\n"; + return 0; + } + + $image->{astrom} = $astromFile; + $image->{cam_path_base} = $camRoot; + + return 1; } @@ -959,9 +1000,10 @@ # resolve_project() # get project specific information -my $last_project = ""; sub resolve_project { my $ipprc = shift; my $project_name = shift; + findTools(); + if (!$project_name) { carp ("project is not defined"); @@ -969,5 +1011,5 @@ } - if ($project_name eq $last_project) { + if ($last_project and ($project_name eq ($last_project->{name}))) { return $last_project; } @@ -977,31 +1019,14 @@ my $verbose = 0; - - my $missing_tools; - my $pstamptool = can_run("pstamptool") or (warn "Can't find pstamptool" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } my $command = "$pstamptool -project -name $project_name"; $command .= " -dbname $dbname" if defined $dbname; $command .= " -dbserver $dbserver" if defined $dbserver; - # run the tool and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; + + my $proj_hash = runToolAndParse($command, $verbose); + if (!$proj_hash or scalar @$proj_hash == 0) { + print STDERR "Project $project_name not found\n"; return undef; } - - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - - my $output = join "", @$stdout_buf; - if (!$output) { - print STDERR "no output returned from $command\n" if $verbose; - return undef; - } - my $proj_hash = parse_md_fast($mdcParser, $output); $last_project = $proj_hash->[0]; @@ -1035,13 +1060,293 @@ sub getCamRunByExpID { my $exp_id = shift; - die "getCamRun: exp_id is nil" if !$exp_id; + &my_die ("getCamRun: exp_id is nil", $PS_EXIT_PROG_ERROR) if !$exp_id; return $camRunByExpIDCache{$exp_id}; } + sub getCamRunByCamID { my $cam_id = shift; - die "getCamRun: cam_id is nil" if !$cam_id; + &my_die ("getCamRun: cam_id is nil", $PS_EXIT_PROG_ERROR) if !$cam_id; return $camRunByCamIDCache{$cam_id}; } + +sub selectComponents { + my $ipprc = shift; + my $imagedb = shift; + my $req_type = shift; + my $stage = shift; + my $rowList = shift; + my $runList = shift; + my $verbose = shift; + my $results = []; + + my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps); + my $npoints = 0; + foreach my $row (@$rowList) { + print $pointsList "$npoints $row->{CENTER_X} $row->{CENTER_Y}\n"; + $npoints++; + # this gets overwitten if an overlapping image is found + $row->{error_code} = $PSTAMP_NO_OVERLAP; + } + close $pointsList; + + # XXX: need to loop over these + # my $run = $runList->[0]; + if (($req_type eq "byid") or ($req_type eq "byexp")) { + my ($last_tess_id, $tess_dir_abs, $astrom_file) = ("", "", ""); + foreach my $run (@$runList) { + my $command = "$dvoImagesAtCoords -coords $pointsListName"; + if (($stage eq "chip") or ($stage eq "raw")) { + # XXX: use file rule and handle cameras where the astrometry is solved at + # the chip stage. + $astrom_file = $run->{cam_path_base} . ".smf"; + my $astrom_file_resolved = $ipprc->file_resolve($astrom_file); + $command .= " -astrom $astrom_file_resolved"; + } else { + my $tess_id = $run->{tess_id}; + if ($tess_id ne $last_tess_id) { + $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id ); + $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs ); + $last_tess_id = $tess_id; + } + $command .= " -D CATDIR $tess_dir_abs" + } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $dvo_verbose); + unless ($success) { + # don't fail if the program exited normally and exit status was PSTAMP_NO_OVERLAP + # # That just means that the coordinate didn't match any image/skycell + if (!WIFEXITED($error_code) || (WEXITSTATUS($error_code) ne $PSTAMP_NO_OVERLAP)) { + print STDERR @$stderr_buf; + my $rc = WIFEXITED($error_code) ? WEXITSTATUS($error_code) : $PS_EXIT_SYS_ERROR; + &my_die( "dvoImagesAtCoords failed: $rc", $rc); + } + } + my %components; + my @lines = split "\n", join "", @$stdout_buf; + foreach my $line (@lines) { + my ($ptnum, undef, undef, $component) = split " ", $line; + my $ref = $components{$component}; + if (!$ref) { + $ref = $components{$component} = []; + } + push @$ref, $ptnum; + my $row = $rowList->[$ptnum]; + # this row found a match + $row->{error_code} = 0; + } + if ($verbose) { + foreach my $c (keys %components) { + my $ref = $components{$c}; + print "component $c contains: "; + foreach my $i (@$ref) { + print "$i "; + } + print "\n"; + } + } + # now find the images for this run + foreach my $c (keys %components) { + my $command; + my $stage_id; + if ($stage eq 'raw') { + $stage_id = $run->{exp_id}; + $command = "$regtool -processedimfile -exp_id $stage_id -class_id $c"; + } elsif ($stage eq 'chip') { + $stage_id = $run->{chip_id}; + $command = "$chiptool -processedimfile -chip_id $stage_id -class_id $c"; + } elsif ($stage eq 'warp') { + $stage_id = $run->{warp_id}; + $command = "$warptool -warped -warp_id $stage_id -skycell_id $c"; + } elsif ($stage eq 'diff') { + $stage_id = $run->{diff_id}; + $command = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $c"; + } + $command .= " -dbname $imagedb"; + my $images = runToolAndParse($command, $verbose); + if (!defined $images) { + print "No components containing coordinates found for ${stage}_id $stage_id\n"; + foreach my $row (@$rowList) { + # XXX: This doesn't seem like the correct error code? + $row->{error_code} = $PSTAMP_NO_OVERLAP; + } + next; + } + if (scalar @$images != 1) { + my $num_images = scalar @$images; + &my_die ("unexpected number of images returned: $num_images\n", $PS_EXIT_PROG_ERROR); + } + my $image = $images->[0]; + + my $ref = $components{$c}; + $image->{row_index} = $ref; + if (($stage eq "raw") or ($stage eq 'chip')) { + $image->{astrom} = $astrom_file; + $image->{cam_id} = $run->{cam_id}; + $image->{cam_path_base} = $run->{cam_path_base}; + } + push @$results, $image; + } + } + } else { + &my_die ("sorry not done with $req_type\n", $PS_EXIT_PROG_ERROR); + } + return $results; +} + +sub filterRuns { + my $stage = shift; + my $need_magic = shift; + my $inputs = shift; + my $verbose = shift; + + if ($inputs and (scalar @$inputs) == 1) { + # one run nothing to do + return $inputs; + } + + my $output = []; + + return $output if (!$inputs or scalar @$inputs == 0); + + my $id_name = $stage . "_id"; + + # input list is "order by exp_id, run_id DESC" run_id is one of (chip_id, warp_id, diff_id) + print "Starting filterRuns\n"; + my $last_exp_id = 0; + my $last_run_id = 0; + my $printed = 0; + foreach my $input (@$inputs) { + my $exp_id = $input->{exp_id}; + my $run_id = $input->{$id_name}; + my $magicked = $input->{magicked}; # this will be either stageRun.magicked or stage%file.magicked + my $state = $input->{state}; + + # can't process run in these states + if (($state eq 'new') or ($state eq 'purged') or ($state eq 'scrubbed')) { + print "skipping ${stage}Run $run_id for exp_id $exp_id in state $state\n"; + next; + } + + $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id)); + + # skip if we need magicked run and this one has never been magicked + if ($need_magic and !$magicked) { + print "skipping ${stage}Run $run_id for exp_id $exp_id not magicked\n" if !$printed; + next; + } + + if (($exp_id == $last_exp_id) and ($run_id != $last_run_id)) { + print "Skipping duplicate ${stage}Run $run_id for $exp_id\n" if !$printed; + next; + } + print "Keeping ${stage}Run $run_id for $exp_id\n"; + push @$output, $input; + $last_exp_id = $exp_id; + $last_run_id = $run_id; + $printed = 0; + } + + my $num_runs = scalar @$output; + print "filterRuns returning $num_runs ${stage}Run\n"; + + return $output; +} + +# run a command that produces metadata output and parse the results into an array of objects +sub runToolAndParse { + my $command = shift; + my $verbose = shift; + + my ($program) = split " ", $command; + $program = basename($program); + + print "Running $command\n" if !$verbose; + my $start_tool = gettimeofday(); + # run the command and parse the output + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + # not sure if we should die here + print STDERR @$stderr_buf; + return undef; + } + + my $now = gettimeofday(); + my $dtime_tool = $now - $start_tool; + print "Time to run $program: $dtime_tool\n"; + + my $buf = join "", @$stdout_buf; + if (!$buf) { + return undef; + } + + my $start_parse = gettimeofday(); + + my $mdcParser = PS::IPP::Metadata::Config->new; + my $results = parse_md_fast($mdcParser, $buf) + or die ("Unable to parse metadata config doc"); + + my $dtime_parse = gettimeofday() - $start_parse; + print "Time to parse results from $program: $dtime_parse\n"; + + return $results; +} + +sub findTools { + return if ($regtool); + + my $missing_tools; + $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); + $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); + $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); + $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); + $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); + $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); + $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); + $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" + and $missing_tools = 1); + $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1); + $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); + if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); + } +} + +# add a row_index array to a set of images with entries for each of the rows in a list +sub setRowRefs { + my $rowList = shift; + my $images = shift; + + my $row_index = []; + for (my $i = 0; $i < scalar @$rowList; $i++) { + push @$row_index, $i; + } + foreach my $image (@$images) { + $image->{row_index} = $row_index; + } +} +# set error_code for an array of rows to a given value +sub setErrorCodes { + my $rowList = shift; + my $code = shift; + foreach my $row (@$rowList) { + $row->{error_code} = $code; + } +} + +sub my_die +{ + my $msg = shift; + my $fault = shift; + + carp $msg; + + # we don't fault the request here pstamp_parser_run.pl handles that if necessary + + return $fault; +} 1; Index: /branches/pap/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- /branches/pap/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 28002) +++ /branches/pap/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 28003) @@ -29,5 +29,5 @@ $PSTAMP_SELECT_UNCONV $PSTAMP_USE_IMFILE_ID - $PSTAMP_WAIT_FOR_UPDATE + $PSTAMP_NO_WAIT_FOR_UPDATE $PSTAMP_SUCCESS $PSTAMP_SYSTEM_ERROR @@ -63,5 +63,5 @@ our $PSTAMP_USE_IMFILE_ID = 16384; -our $PSTAMP_WAIT_FOR_UPDATE = 32768; +our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768; # job and result codes @@ -145,8 +145,5 @@ } if (@$stdout_buf) { - my $table = $mdcParser->parse(join "", @$stdout_buf) or - die("Unable to parse metdata config doc"); - - $rows = parse_md_list($table); + $rows = parse_md_fast($mdcParser, join "", @$stdout_buf); } else { print STDERR "$request_file_name contains empty request table\n"; @@ -180,3 +177,31 @@ return (\%header, \%req_specs); } + +sub parse_md_fast { + my $mdcParser = shift; + my $input = shift; + my $output = (); + + my @whole = split /\n/, $input; + my @single = (); + + my $n; + while ( ($n = @whole) > 0) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join("\n", @single ) ) ) or + print STDERR "Unable to parse metdata config doc" and return undef; +# my $num = @$list; +# print STDERR "list has $num elments\n"; + push @$output, $list->[0]; + + @single = (); + } + } + return $output; +} + 1; Index: /branches/pap/dbconfig/changes.txt =================================================================== --- /branches/pap/dbconfig/changes.txt (revision 28002) +++ /branches/pap/dbconfig/changes.txt (revision 28003) @@ -1554,5 +1554,5 @@ note VARCHAR(255), -- note registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered - PRIMARY KEY(dqstats_id), + PRIMARY KEY(ss_id), KEY(state), KEY(label), @@ -1586,4 +1586,32 @@ KEY(fault), KEY(quality), - FOREIGN KEY(ss_id) REFERENCES staticskyRun(ss_id), -) ENGINE=innodb DEFAULT CHARSET=latin1; + FOREIGN KEY(ss_id) REFERENCES staticskyRun(ss_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +ALTER TABLE pstampDependent ADD COLUMN component VARCHAR(64) after stage_id; +ALTER TABLE pstampDependent CHANGE COLUMN no_magic need_magic TINYINT; +ALTER TABLE pstampDependent ADD COLUMN fault SMALLINT after need_magic; +ALTER TABLE pstampDependent ADD COLUMN outdir VARCHAR(255) after imagedb; +ALTER TABLE pstampDependent ADD KEY (state); +ALTER TABLE pstampDependent ADD KEY (stage); +ALTER TABLE pstampDependent ADD KEY (stage_id); +ALTER TABLE pstampDependent ADD KEY (component); +ALTER TABLE pstampDependent ADD KEY (imagedb); + +ALTER TABLE magicDSRun ADD KEY(stage); +ALTER TABLE magicDSRun ADD KEY(stage_id); + +ALTER TABLE distRun ADD KEY(stage), ADD KEY(stage_id), ADD KEY(magic_ds_id), ADD KEY(fault), ADD KEY(data_group); +ALTER TABLE distTarget ADD KEY(stage), ADD KEY(dist_group), ADD KEY(filter), ADD KEY(state); + +CREATE TABLE Label ( + label VARCHAR(64), + priority INT, + active TINYINT, + comment VARCHAR(80), + PRIMARY KEY(label), + KEY(priority), + KEY(active) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +ALTER TABLE pstampRequest ADD COLUMN outdir VARCHAR(64) after uri; Index: /branches/pap/dbconfig/ipp.m4 =================================================================== --- /branches/pap/dbconfig/ipp.m4 (revision 28002) +++ /branches/pap/dbconfig/ipp.m4 (revision 28003) @@ -21,4 +21,5 @@ include(stack.md) include(det.md) +include(label.md) include(magic.md) include(calibration.md) Index: /branches/pap/dbconfig/label.md =================================================================== --- /branches/pap/dbconfig/label.md (revision 28003) +++ /branches/pap/dbconfig/label.md (revision 28003) @@ -0,0 +1,6 @@ +Label METADATA + label STR 64 # Primary Key + priority S32 0 + active BOOL t + comment STR 80 +END Index: /branches/pap/dbconfig/pstamp.md =================================================================== --- /branches/pap/dbconfig/pstamp.md (revision 28002) +++ /branches/pap/dbconfig/pstamp.md (revision 28003) @@ -30,4 +30,5 @@ outProduct STR 64 uri STR 255 + outdir STR 255 fault S32 0 END @@ -51,6 +52,9 @@ stage STR 64 stage_id S64 0 + component STR 64 imagedb STR 64 rlabel STR 64 - no_magic BOOL f + need_magic BOOL f + outdir STR 255 + fault S16 0 END Index: /branches/pap/dbconfig/staticsky.md =================================================================== --- /branches/pap/dbconfig/staticsky.md (revision 28002) +++ /branches/pap/dbconfig/staticsky.md (revision 28003) @@ -2,5 +2,5 @@ staticskyRun METADATA - ss_id S64 0 # Primary Key AUTO_INCREMENT + sky_id S64 0 # Primary Key AUTO_INCREMENT state STR 64 # Key workdir STR 255 @@ -14,10 +14,10 @@ staticskyInput METADATA - ss_id S64 0 # Primary Key fkey(ss_id) ref staticskyRun(ss_id) + sky_id S64 0 # Primary Key fkey(sky_id) ref staticskyRun(sky_id) stack_id S64 0 # Primary Key fkey(stack_id) ref stackSumSkyfile(stack_id) END -staticSkyResult METADATA - ss_id S64 0 # Primary Key fkey(ss_id) ref staticskyRun(ss_id) +staticskyResult METADATA + sky_id S64 0 # Primary Key fkey(sky_id) ref staticskyRun(sky_id) path_base STR 255 dtime_phot F32 0.0 Index: /branches/pap/doc/pslib/psLibSDRS.tex =================================================================== --- /branches/pap/doc/pslib/psLibSDRS.tex (revision 28002) +++ /branches/pap/doc/pslib/psLibSDRS.tex (revision 28003) @@ -7449,8 +7449,10 @@ \end{prototype} -The following function changes the given rotation to its inverse: - -\begin{prototype} -psSphereRot *psSphereRotInvert(psSphereRot *rot); +The following function constructs a the \code{psSphereRot} with the +inverse rotation of the given euler angles. This is equivalent to +applying \code{psSphereRotConjugate} on a \code{psSphereRot} object: + +\begin{prototype} +psSphereRot *psSphereRotInvert(double alphaP, double deltaP, double phiP); \end{prototype} Index: /branches/pap/ippMonitor/Makefile.in =================================================================== --- /branches/pap/ippMonitor/Makefile.in (revision 28002) +++ /branches/pap/ippMonitor/Makefile.in (revision 28003) @@ -13,5 +13,8 @@ PROGRAMS = \ -$(DESTBIN)/skyplot.dvo +$(DESTBIN)/skyplot.dvo \ +$(DESTBIN)/getLabels.pl \ +$(DESTBIN)/checkServer.pl \ +$(DESTBIN)/getServerStatus.pl RAWSRC = \ @@ -43,5 +46,7 @@ $(DESTWWW)/simplePlotraw.php \ $(DESTWWW)/skyplot.php \ -$(DESTWWW)/getimage.php +$(DESTWWW)/getimage.php \ +$(DESTWWW)/czartool.php \ + DEFSRC = \ Index: /branches/pap/ippMonitor/def/camIQstats.d =================================================================== --- /branches/pap/ippMonitor/def/camIQstats.d (revision 28002) +++ /branches/pap/ippMonitor/def/camIQstats.d (revision 28003) @@ -20,4 +20,6 @@ FIELD rawExp.exp_name, 5, %s, Exp Name FIELD chipRun.label, 7, %s, Label +FIELD chipRun.data_group, 7, %s, data grp +FIELD chipRun.dist_group, 7, %s, dist grp FIELD rawExp.dateobs, 19, %T, Date/Time FIELD rawExp.filter, 10, %s, FILTER Index: /branches/pap/ippMonitor/def/camPendingExp.d =================================================================== --- /branches/pap/ippMonitor/def/camPendingExp.d (revision 28002) +++ /branches/pap/ippMonitor/def/camPendingExp.d (revision 28003) @@ -22,4 +22,6 @@ FIELD camRun.state, 5, %s, State FIELD camRun.label, 5, %s, Label +FIELD camRun.data_group, 5, %s, data grp +FIELD camRun.dist_group, 5, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/camProcessedExp.d =================================================================== --- /branches/pap/ippMonitor/def/camProcessedExp.d (revision 28002) +++ /branches/pap/ippMonitor/def/camProcessedExp.d (revision 28003) @@ -30,4 +30,6 @@ FIELD camRun.state, 5, %s, state, value, camProcessedImfile.php, ARG4 FIELD camRun.label, 5, %s, Label +FIELD camRun.data_group, 5, %s, data grp +FIELD camRun.dist_group, 5, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/camProcessedExp_Images.d =================================================================== --- /branches/pap/ippMonitor/def/camProcessedExp_Images.d (revision 28002) +++ /branches/pap/ippMonitor/def/camProcessedExp_Images.d (revision 28003) @@ -33,4 +33,6 @@ FIELD camRun.cam_id, 5, %d, Cam ID, value, camProcessedExp_Images.php, ARG3 FIELD camRun.label, 5, %s, Label +FIELD camRun.data_group, 5, %s, data grp +FIELD camRun.dist_group, 5, %s, dist grp FIELD *, 8, %s, image, image=JPEG2, camProcessedImfile.php, ARG4 FIELD rawExp.telescope, 10, %s, Telescope Index: /branches/pap/ippMonitor/def/camProcessedExp_NoImages.d =================================================================== --- /branches/pap/ippMonitor/def/camProcessedExp_NoImages.d (revision 28002) +++ /branches/pap/ippMonitor/def/camProcessedExp_NoImages.d (revision 28003) @@ -30,4 +30,6 @@ FIELD camRun.state, 5, %s, state, value, camProcessedImfile.php, ARG4 FIELD camRun.label, 5, %s, label +FIELD camRun.data_group, 5, %s, data grp +FIELD camRun.dist_group, 5, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/camProcessedExp_failure.d =================================================================== --- /branches/pap/ippMonitor/def/camProcessedExp_failure.d (revision 28002) +++ /branches/pap/ippMonitor/def/camProcessedExp_failure.d (revision 28003) @@ -29,4 +29,6 @@ FIELD camRun.state, 5, %s, state, value, camProcessedImfile.php, ARG4 FIELD camRun.label, 5, %s, label +FIELD camRun.data_group, 5, %s, data grp +FIELD camRun.dist_group, 5, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/camProcessedImfile.d =================================================================== --- /branches/pap/ippMonitor/def/camProcessedImfile.d (revision 28002) +++ /branches/pap/ippMonitor/def/camProcessedImfile.d (revision 28003) @@ -27,4 +27,6 @@ FIELD camRun.cam_id, 5, %s, Cam ID, value, camProcessedExp.php, ARG3 FIELD camRun.label, 5, %s, Label +FIELD camRun.data_group, 5, %s, data grp +FIELD camRun.dist_group, 5, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/camStageExp.d =================================================================== --- /branches/pap/ippMonitor/def/camStageExp.d (revision 28002) +++ /branches/pap/ippMonitor/def/camStageExp.d (revision 28003) @@ -21,4 +21,6 @@ FIELD camRun.state, 5, %s, state FIELD camRun.label, 5, %s, label +FIELD camRun.data_group, 5, %s, data grp +FIELD camRun.dist_group, 5, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/chipIQstats.d =================================================================== --- /branches/pap/ippMonitor/def/chipIQstats.d (revision 28002) +++ /branches/pap/ippMonitor/def/chipIQstats.d (revision 28003) @@ -20,4 +20,6 @@ FIELD chipRun.chip_id, 5, %d, Ch ID, value, chipProcessedImfile.php, ARG2 FIELD chipRun.label, 7, %s, Label +FIELD chipRun.data_group, 7, %s, data grp +FIELD chipRun.dist_group, 7, %s, dist grp FIELD rawExp.dateobs, 19, %T, Date/Time FIELD rawExp.filter, 10, %s, FILTER Index: /branches/pap/ippMonitor/def/chipPendingExp.d =================================================================== --- /branches/pap/ippMonitor/def/chipPendingExp.d (revision 28002) +++ /branches/pap/ippMonitor/def/chipPendingExp.d (revision 28003) @@ -20,4 +20,6 @@ FIELD chipRun.state, 7, %s, State FIELD chipRun.label, 7, %s, Label +FIELD chipRun.data_group, 7, %s, data grp +FIELD chipRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/chipPendingImfile.d =================================================================== --- /branches/pap/ippMonitor/def/chipPendingImfile.d (revision 28002) +++ /branches/pap/ippMonitor/def/chipPendingImfile.d (revision 28003) @@ -25,4 +25,6 @@ FIELD chipRun.state, 7, %s, State FIELD chipRun.label, 7, %s, Label +FIELD chipRun.data_group, 7, %s, data grp +FIELD chipRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/chipProcessedExp.d =================================================================== --- /branches/pap/ippMonitor/def/chipProcessedExp.d (revision 28002) +++ /branches/pap/ippMonitor/def/chipProcessedExp.d (revision 28003) @@ -20,4 +20,6 @@ FIELD chipRun.state, 7, %s, State FIELD chipRun.label, 7, %s, Label +FIELD chipRun.data_group, 7, %s, data grp +FIELD chipRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/chipProcessedImfile.d =================================================================== --- /branches/pap/ippMonitor/def/chipProcessedImfile.d (revision 28002) +++ /branches/pap/ippMonitor/def/chipProcessedImfile.d (revision 28003) @@ -30,4 +30,6 @@ FIELD chipRun.state, 7, %s, State, value, chipProcessedImfile.php, ARG3 FIELD chipRun.label, 7, %s, Label +FIELD chipRun.data_group, 7, %s, data grp +FIELD chipRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/chipProcessedImfile_failure.d =================================================================== --- /branches/pap/ippMonitor/def/chipProcessedImfile_failure.d (revision 28002) +++ /branches/pap/ippMonitor/def/chipProcessedImfile_failure.d (revision 28003) @@ -30,4 +30,6 @@ FIELD chipRun.state, 7, %s, State, value, chipProcessedImfile_failure.php, ARG3 FIELD chipRun.label, 7, %s, Label +FIELD chipRun.data_group, 7, %s, data grp +FIELD chipRun.dist_group, 7, %s, dist grp FIELD chipProcessedImfile.fault, 2, %d, Fault FIELD rawExp.telescope, 10, %s, Telescope Index: /branches/pap/ippMonitor/def/chipStageExp.d =================================================================== --- /branches/pap/ippMonitor/def/chipStageExp.d (revision 28002) +++ /branches/pap/ippMonitor/def/chipStageExp.d (revision 28003) @@ -18,4 +18,6 @@ FIELD chipRun.state, 7, %s, State FIELD chipRun.label, 7, %s, Label +FIELD chipRun.data_group, 7, %s, data grp +FIELD chipRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/destreakRun.d =================================================================== --- /branches/pap/ippMonitor/def/destreakRun.d (revision 28002) +++ /branches/pap/ippMonitor/def/destreakRun.d (revision 28003) @@ -19,5 +19,5 @@ FIELD magicDSRun.cam_id, 5, %d, Camera ID FIELD magicDSRun.label, 10, %s, Label -FIELD magicDSRun.data_group, 8, %s, Data Group +FIELD magicDSRun.data_group, 10, %s, data grp FIELD magicDSRun.outroot, 10, %s, Out Root FIELD magicDSRun.recoveryroot, 5, %d, Recov. Root Index: /branches/pap/ippMonitor/def/diffFailedSkyfile.d =================================================================== --- /branches/pap/ippMonitor/def/diffFailedSkyfile.d (revision 28002) +++ /branches/pap/ippMonitor/def/diffFailedSkyfile.d (revision 28003) @@ -16,4 +16,6 @@ FIELD diffSkyfile.skycell_id, 5, %s, Skycell ID FIELD diffRun.label, 5, %s, Label +FIELD diffRun.data_group, 5, %s, data grp +FIELD diffRun.dist_group, 5, %s, dist grp # FIELD diffRun.filter, 5, %s, Filter FIELD diffRun.tess_id, 5, %s, Tess ID Index: /branches/pap/ippMonitor/def/diffInputSkyfile.d =================================================================== --- /branches/pap/ippMonitor/def/diffInputSkyfile.d (revision 28002) +++ /branches/pap/ippMonitor/def/diffInputSkyfile.d (revision 28003) @@ -12,4 +12,6 @@ FIELD diffRun.state, 7, %s, state FIELD diffRun.label, 7, %s, label +FIELD diffRun.data_group, 7, %s, data grp +FIELD diffRun.dist_group, 7, %s, dist grp FIELD diffRun.magicked, 4, %d, Magicked FIELD diffInputSkyfile.warp1, 5, %d, warp 1 Index: /branches/pap/ippMonitor/def/diffProcessedSkyfile.d =================================================================== --- /branches/pap/ippMonitor/def/diffProcessedSkyfile.d (revision 28002) +++ /branches/pap/ippMonitor/def/diffProcessedSkyfile.d (revision 28003) @@ -17,4 +17,6 @@ FIELD diffSkyfile.skycell_id, 4, %s, Skycell ID FIELD diffRun.label, 5, %s, Label +FIELD diffRun.data_group, 5, %s, data grp +FIELD diffRun.dist_group, 5, %s, dist grp # FIELD diffRun.filter, 5, %s, Filter FIELD diffRun.tess_id, 5, %s, Tess ID Index: /branches/pap/ippMonitor/def/diffProcessedSkyfile_Images.d =================================================================== --- /branches/pap/ippMonitor/def/diffProcessedSkyfile_Images.d (revision 28002) +++ /branches/pap/ippMonitor/def/diffProcessedSkyfile_Images.d (revision 28003) @@ -22,4 +22,6 @@ FIELD diffSkyfile.skycell_id, 5, %s, Skycell ID FIELD diffRun.label, 5, %s, Label +FIELD diffRun.data_group, 5, %s, data grp +FIELD diffRun.dist_group, 5, %s, dist grp FIELD *, 8, %s, image, image=JPEG2, diffProcessedSkyfile_Images.php, ARG7 # FIELD diffRun.filter, 5, %s, Filter Index: /branches/pap/ippMonitor/def/diffRun.d =================================================================== --- /branches/pap/ippMonitor/def/diffRun.d (revision 28002) +++ /branches/pap/ippMonitor/def/diffRun.d (revision 28003) @@ -8,4 +8,6 @@ FIELD state, 7, %s, state FIELD label, 7, %s, label +FIELD data_group, 7, %s, data grp +FIELD dist_group, 7, %s, dist grp FIELD magicked, 4, %d, Magicked FIELD tess_id, 8, %s, tess ID Index: /branches/pap/ippMonitor/def/distRun.d =================================================================== --- /branches/pap/ippMonitor/def/distRun.d (revision 28002) +++ /branches/pap/ippMonitor/def/distRun.d (revision 28003) @@ -15,4 +15,5 @@ FIELD distRun.magic_ds_id, 5, %d, Magic DS ID FIELD distRun.label, 5, %s, Label +FIELD distRun.data_group, 5, %s, data grp FIELD distRun.outroot, 5, %s, Outroot FIELD distRun.outdir, 5, %s, Outdir Index: /branches/pap/ippMonitor/def/distSummary.d =================================================================== --- /branches/pap/ippMonitor/def/distSummary.d (revision 28002) +++ /branches/pap/ippMonitor/def/distSummary.d (revision 28003) @@ -6,4 +6,5 @@ # field size format name show link to extras FIELD distRun.label, 15, %s, Label +FIELD distRun.data_group, 15, %s, data grp FIELD distRun.stage, 5, %s, Stage FIELD distRun.state, 5, %s, State Index: /branches/pap/ippMonitor/def/fakePendingExp.d =================================================================== --- /branches/pap/ippMonitor/def/fakePendingExp.d (revision 28002) +++ /branches/pap/ippMonitor/def/fakePendingExp.d (revision 28003) @@ -26,4 +26,6 @@ FIELD fakeRun.state, 7, %s, state FIELD fakeRun.label, 7, %s, label +FIELD fakeRun.data_group, 7, %s, data grp +FIELD fakeRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/fakePendingImfile.d =================================================================== --- /branches/pap/ippMonitor/def/fakePendingImfile.d (revision 28002) +++ /branches/pap/ippMonitor/def/fakePendingImfile.d (revision 28003) @@ -27,4 +27,6 @@ FIELD fakeRun.state, 7, %s, State FIELD fakeRun.label, 7, %s, label +FIELD fakeRun.data_group, 7, %s, data grp +FIELD fakeRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/fakeProcessedExp.d =================================================================== --- /branches/pap/ippMonitor/def/fakeProcessedExp.d (revision 28002) +++ /branches/pap/ippMonitor/def/fakeProcessedExp.d (revision 28003) @@ -26,4 +26,6 @@ FIELD fakeRun.state, 7, %s, state FIELD fakeRun.label, 7, %s, label +FIELD fakeRun.data_group, 7, %s, data grp +FIELD fakeRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/fakeProcessedImfile.d =================================================================== --- /branches/pap/ippMonitor/def/fakeProcessedImfile.d (revision 28002) +++ /branches/pap/ippMonitor/def/fakeProcessedImfile.d (revision 28003) @@ -34,4 +34,6 @@ FIELD fakeRun.state, 7, %s, State, value, fakeProcessedImfile.php, ARG5 FIELD fakeRun.label, 7, %s, Label +FIELD fakeRun.data_group, 7, %s, data grp +FIELD fakeRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/fakeProcessedImfile_failure.d =================================================================== --- /branches/pap/ippMonitor/def/fakeProcessedImfile_failure.d (revision 28002) +++ /branches/pap/ippMonitor/def/fakeProcessedImfile_failure.d (revision 28003) @@ -34,4 +34,6 @@ FIELD fakeRun.state, 7, %s, State, value, fakeProcessedImfile_failure.php, ARG5 FIELD fakeRun.label, 7, %s, Label +FIELD fakeRun.data_group, 7, %s, data grp +FIELD fakeRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/fakeStageExp.d =================================================================== --- /branches/pap/ippMonitor/def/fakeStageExp.d (revision 28002) +++ /branches/pap/ippMonitor/def/fakeStageExp.d (revision 28003) @@ -24,4 +24,6 @@ FIELD fakeRun.state, 7, %s, state FIELD fakeRun.label, 7, %s, label +FIELD fakeRun.data_group, 7, %s, data grp +FIELD fakeRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/flatcorrCamera.d =================================================================== --- /branches/pap/ippMonitor/def/flatcorrCamera.d (revision 28002) +++ /branches/pap/ippMonitor/def/flatcorrCamera.d (revision 28003) @@ -19,4 +19,6 @@ FIELD camRun.state, 7, %s, State FIELD chipRun.label, 7, %s, Label +FIELD chipRun.data_group, 7, %s, data grp +FIELD chipRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/flatcorrChip.d =================================================================== --- /branches/pap/ippMonitor/def/flatcorrChip.d (revision 28002) +++ /branches/pap/ippMonitor/def/flatcorrChip.d (revision 28003) @@ -18,4 +18,6 @@ FIELD chipRun.state, 7, %s, State FIELD chipRun.label, 7, %s, Label +FIELD chipRun.data_group, 7, %s, data grp +FIELD chipRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/magicFailedMasks.d =================================================================== --- /branches/pap/ippMonitor/def/magicFailedMasks.d (revision 28002) +++ /branches/pap/ippMonitor/def/magicFailedMasks.d (revision 28003) @@ -12,4 +12,5 @@ FIELD magicRun.diff_id, 5, %d, Diff ID FIELD magicRun.label, 5, %s, Label +FIELD magicRun.data_group, 5, %s, data grp FIELD magicRun.inverse, 5, %d, Inverse FIELD magicRun.state, 5, %s, State Index: /branches/pap/ippMonitor/def/magicFailedNodes.d =================================================================== --- /branches/pap/ippMonitor/def/magicFailedNodes.d (revision 28002) +++ /branches/pap/ippMonitor/def/magicFailedNodes.d (revision 28003) @@ -14,4 +14,5 @@ FIELD magicRun.diff_id, 5, %d, Diff ID FIELD magicRun.label, 5, %s, Label +FIELD magicRun.data_group, 5, %s, data grp FIELD magicRun.inverse, 5, %d, Inverse FIELD magicRun.state, 5, %s, State Index: /branches/pap/ippMonitor/def/magicFailedTrees.d =================================================================== --- /branches/pap/ippMonitor/def/magicFailedTrees.d (revision 28002) +++ /branches/pap/ippMonitor/def/magicFailedTrees.d (revision 28003) @@ -11,4 +11,5 @@ FIELD diff_id, 5, %d, Diff ID FIELD label, 5, %s, Label +FIELD data_group, 5, %s, Data Grp FIELD inverse, 5, %d, Inverse FIELD state, 5, %s, State Index: /branches/pap/ippMonitor/def/magicProcessedMasks.d =================================================================== --- /branches/pap/ippMonitor/def/magicProcessedMasks.d (revision 28002) +++ /branches/pap/ippMonitor/def/magicProcessedMasks.d (revision 28003) @@ -11,4 +11,5 @@ FIELD magicRun.diff_id, 5, %d, Diff ID FIELD magicRun.label, 5, %s, Label +FIELD magicRun.data_group, 5, %s, data grp FIELD magicRun.inverse, 5, %d, Inverse FIELD magicRun.state, 5, %s, State Index: /branches/pap/ippMonitor/def/magicProcessedMasks_Images.d =================================================================== --- /branches/pap/ippMonitor/def/magicProcessedMasks_Images.d (revision 28002) +++ /branches/pap/ippMonitor/def/magicProcessedMasks_Images.d (revision 28003) @@ -19,4 +19,5 @@ # FIELD *, 8, %s, image, image=JPEG2, diffProcessedSkyfile_Images.php, ARG7 FIELD magicRun.label, 5, %s, Label +FIELD magicRun.data_group, 5, %s, data grp FIELD magicRun.inverse, 5, %d, Inverse FIELD magicRun.state, 5, %s, State Index: /branches/pap/ippMonitor/def/magicProcessedNodes.d =================================================================== --- /branches/pap/ippMonitor/def/magicProcessedNodes.d (revision 28002) +++ /branches/pap/ippMonitor/def/magicProcessedNodes.d (revision 28003) @@ -13,4 +13,5 @@ FIELD magicRun.diff_id, 5, %d, Diff ID FIELD magicRun.label, 5, %s, Label +FIELD magicRun.data_group, 5, %s, data grp FIELD magicRun.inverse, 5, %d, Inverse FIELD magicRun.state, 5, %s, State Index: /branches/pap/ippMonitor/def/magicRun.d =================================================================== --- /branches/pap/ippMonitor/def/magicRun.d (revision 28002) +++ /branches/pap/ippMonitor/def/magicRun.d (revision 28003) @@ -9,4 +9,5 @@ FIELD diff_id, 5, %d, Diff ID FIELD label, 5, %s, Label +FIELD data_group, 5, %s, Data Grp FIELD inverse, 5, %d, Inverse FIELD state, 5, %s, State Index: /branches/pap/ippMonitor/def/magicTrees.d =================================================================== --- /branches/pap/ippMonitor/def/magicTrees.d (revision 28002) +++ /branches/pap/ippMonitor/def/magicTrees.d (revision 28003) @@ -11,4 +11,5 @@ FIELD magicRun.diff_id, 5, %d, Diff ID FIELD magicRun.label, 5, %s, Label +FIELD magicRun.data_group, 5, %s, data grp FIELD magicRun.inverse, 5, %d, Inverse FIELD magicRun.state, 5, %s, State Index: /branches/pap/ippMonitor/def/stackFailedSkyfile.d =================================================================== --- /branches/pap/ippMonitor/def/stackFailedSkyfile.d (revision 28002) +++ /branches/pap/ippMonitor/def/stackFailedSkyfile.d (revision 28003) @@ -16,4 +16,6 @@ FIELD stackRun.skycell_id, 5, %s, Skycell ID FIELD stackRun.label, 5, %s, Label +FIELD stackRun.data_group, 5, %s, data grp +FIELD stackRun.dist_group, 5, %s, dist grp FIELD stackRun.filter, 5, %s, Filter FIELD stackRun.tess_id, 5, %s, Tess ID Index: /branches/pap/ippMonitor/def/stackInputSkyfile.d =================================================================== --- /branches/pap/ippMonitor/def/stackInputSkyfile.d (revision 28002) +++ /branches/pap/ippMonitor/def/stackInputSkyfile.d (revision 28003) @@ -34,4 +34,6 @@ FIELD stackRun.state, 7, %s, State FIELD stackRun.label, 7, %s, Label +FIELD stackRun.data_group, 7, %s, data grp +FIELD stackRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/stackProcessedSkyfile.d =================================================================== --- /branches/pap/ippMonitor/def/stackProcessedSkyfile.d (revision 28002) +++ /branches/pap/ippMonitor/def/stackProcessedSkyfile.d (revision 28003) @@ -17,4 +17,6 @@ FIELD stackRun.skycell_id, 5, %s, Skycell ID FIELD stackRun.label, 5, %s, Label +FIELD stackRun.data_group, 5, %s, data grp +FIELD stackRun.dist_group, 5, %s, dist grp FIELD stackRun.filter, 5, %s, Filter FIELD stackRun.tess_id, 5, %s, Tess ID Index: /branches/pap/ippMonitor/def/stackProcessedSkyfile_Images.d =================================================================== --- /branches/pap/ippMonitor/def/stackProcessedSkyfile_Images.d (revision 28002) +++ /branches/pap/ippMonitor/def/stackProcessedSkyfile_Images.d (revision 28003) @@ -21,4 +21,6 @@ FIELD stackRun.skycell_id, 5, %s, Skycell ID FIELD stackRun.label, 5, %s, Label +FIELD stackRun.data_group, 5, %s, data grp +FIELD stackRun.dist_group, 5, %s, dist grp FIELD *, 8, %s, image, image=JPEG2, stackProcessedSkyfile_Images.php, ARG7 FIELD stackRun.filter, 5, %s, Filter Index: /branches/pap/ippMonitor/def/stackRun.d =================================================================== --- /branches/pap/ippMonitor/def/stackRun.d (revision 28002) +++ /branches/pap/ippMonitor/def/stackRun.d (revision 28003) @@ -9,4 +9,6 @@ FIELD filter, 5, %s, Filter FIELD label, 5, %s, Label +FIELD data_group, 5, %s, Data grp +FIELD dist_group, 5, %s, Dist grp FIELD tess_id, 5, %s, Tess ID FIELD state, 5, %s, State Index: /branches/pap/ippMonitor/def/warpFailedSkyfiles.d =================================================================== --- /branches/pap/ippMonitor/def/warpFailedSkyfiles.d (revision 28002) +++ /branches/pap/ippMonitor/def/warpFailedSkyfiles.d (revision 28003) @@ -36,4 +36,6 @@ FIELD warpRun.state, 7, %s, state, value, warpFailedSkyfiles.php, ARG7 FIELD warpRun.label, 7, %s, label +FIELD warpRun.data_group, 7, %s, data grp +FIELD warpRun.dist_group, 7, %s, dist grp FIELD warpSkyfile.tess_id, 10, %s, tess FIELD warpSkyfile.good_frac, 5, %s, good_frac Index: /branches/pap/ippMonitor/def/warpProcessedSkyfiles.d =================================================================== --- /branches/pap/ippMonitor/def/warpProcessedSkyfiles.d (revision 28002) +++ /branches/pap/ippMonitor/def/warpProcessedSkyfiles.d (revision 28003) @@ -39,4 +39,6 @@ FIELD warpRun.state, 7, %s, state, value, warpProcessedSkyfiles.php, ARG7 FIELD warpRun.label, 7, %s, label, value +FIELD warpRun.data_group, 7, %s, data grp +FIELD warpRun.dist_group, 7, %s, dist grp FIELD warpSkyfile.tess_id, 10, %s, tess FIELD warpSkyfile.good_frac, 5, %s, good_frac Index: /branches/pap/ippMonitor/def/warpProcessedSkyfiles_Images.d =================================================================== --- /branches/pap/ippMonitor/def/warpProcessedSkyfiles_Images.d (revision 28002) +++ /branches/pap/ippMonitor/def/warpProcessedSkyfiles_Images.d (revision 28003) @@ -43,4 +43,6 @@ FIELD warpRun.state, 7, %s, state, value, warpProcessedSkyfiles.php, ARG7 FIELD warpRun.label, 7, %s, label, value +FIELD warpRun.data_group, 7, %s, label, data grp +FIELD warpRun.dist_group, 7, %s, label, dist grp FIELD warpSkyfile.tess_id, 10, %s, tess FIELD warpSkyfile.good_frac, 5, %s, good_frac Index: /branches/pap/ippMonitor/def/warpSkycellMap.d =================================================================== --- /branches/pap/ippMonitor/def/warpSkycellMap.d (revision 28002) +++ /branches/pap/ippMonitor/def/warpSkycellMap.d (revision 28003) @@ -20,4 +20,6 @@ FIELD warpRun.state, 5, %s, State FIELD warpRun.label, 5, %s, Label +FIELD warpRun.data_group, 5, %s, data grp +FIELD warpRun.dist_group, 5, %s, dist grp FIELD warpSkyCellMap.skycell_id, 5, %s, Skycell ID FIELD warpSkyCellMap.tess_id, 5, %s, Tessellation ID Index: /branches/pap/ippMonitor/def/warpStageExp.d =================================================================== --- /branches/pap/ippMonitor/def/warpStageExp.d (revision 28002) +++ /branches/pap/ippMonitor/def/warpStageExp.d (revision 28003) @@ -27,4 +27,6 @@ FIELD warpRun.state, 7, %s, state FIELD warpRun.label, 7, %s, label +FIELD warpRun.data_group, 7, %s, data grp +FIELD warpRun.dist_group, 7, %s, dist grp FIELD rawExp.telescope, 10, %s, Telescope FIELD rawExp.camera, 10, %s, Camera Index: /branches/pap/ippMonitor/def/warpStageSkyfileInputs.d =================================================================== --- /branches/pap/ippMonitor/def/warpStageSkyfileInputs.d (revision 28002) +++ /branches/pap/ippMonitor/def/warpStageSkyfileInputs.d (revision 28003) @@ -40,4 +40,6 @@ FIELD warpRun.state, 7, %s, state FIELD warpRun.label, 7, %s, label +FIELD warpRun.data_group, 7, %s, data grp +FIELD warpRun.dist_group, 7, %s, dist grp FIELD warpSkyCellMap.skycell_id, 10, %s, skycell FIELD warpSkyCellMap.tess_id, 10, %s, tess Index: /branches/pap/ippMonitor/def/warpStageSkyfiles.d =================================================================== --- /branches/pap/ippMonitor/def/warpStageSkyfiles.d (revision 28002) +++ /branches/pap/ippMonitor/def/warpStageSkyfiles.d (revision 28003) @@ -30,4 +30,6 @@ FIELD warpRun.state, 7, %s, state FIELD warpRun.label, 7, %s, label +FIELD warpRun.data_group, 7, %s, data grp +FIELD warpRun.dist_group, 7, %s, dist grp FIELD warpSkyCellMap.tess_id, 10, %s, tess FIELD warpSkyCellMap.fault, 10, %s, fault Index: /branches/pap/ippMonitor/raw/czartool.php =================================================================== --- /branches/pap/ippMonitor/raw/czartool.php (revision 28003) +++ /branches/pap/ippMonitor/raw/czartool.php (revision 28003) @@ -0,0 +1,315 @@ +"; + for ($i = 0; $i < count($distLabels); $i++) { + echo "$distLabels[$i]
"; + } +} + +$states=array("full","new","drop","wait"); +$stages=array("chip","cam","fake","warp","stack","diff","magic","destreak","dist"); +$servers=array("addstar", "cleanup", "detrend", "distribution", "pstamp", "publishing", "registration", "replication", "stdscience", "summitcopy"); +$separator = "-----"; +// set up the form +echo "
\n"; +echo "\n"; +echo "\n"; + +echo "\n"; +echo "\n"; +echo "\n"; +echo "
\n"; + + +$stateChosen = 0; +foreach ($states as &$state) { + + if ($userSelection == $state) { + showAllLabels($pass, $proj, $db, $stdsLabels, $distLabels, $stages, $states, $state); + $stateChosen=1; + break; + } +} + +$serverChosen = 0; +foreach ($servers as &$server) { + + if ($userSelection == $server) { + showServerStatus($server); + $serverChosen=1; + break; + } +} +if (!$stateChosen && !$serverChosen) { + + if ($userSelection == "servers") {showAllServerStatus($servers); }// TODO + elseif ($userSelection == $separator) {} + else { showOneLabel($pass, $proj, $db, $userSelection, $stages, $states);} + +} +menu_end(); + +########################################################################### +# +# Shows the status of the provided pantasks server +# +########################################################################### +function showServerStatus($server) { + + echo "

Status for $server server

"; + + $results=array(); + exec("getServerStatus.pl -s $server", $results, $status); + + foreach ($results as &$line) { + echo "
\n";
+        echo "$line\n";
+        echo "
\n"; + + } + +} + +########################################################################### +# +# Checks the status of all the pantasks servers +# +########################################################################### +function showAllServerStatus($servers) { + + echo "

Status for all pantasks servers

"; + + // set up table columns + $class = "list"; + echo "\n"; + echo "\n"; + write_header_cell($class, "Server"); + write_header_cell($class, "Alive?"); + write_header_cell($class, "Scheduler running?"); + echo "\n"; + + foreach ($servers as &$server) { + + $results=array(); + exec("checkServer.pl -s $server", $results, $status); + + echo "\n"; + write_table_cell($class, '%s', "", $server); + write_table_cell($class, '%s', "", $results[0]); + write_table_cell($class, '%s', "", $results[1]); + echo "\n"; + } + + echo "
\n"; +} + + +########################################################################### +# +# Checks one label and prints results in a table +# +########################################################################### +function showOneLabel($pass, $proj, $db, $label, $stages, $states) { + + echo "

Current status for label '$label' (any faults are shown in parentheses)

"; + + // set up table columns + $class = "list"; + echo "\n"; + echo "\n"; + write_header_cell($class, "state"); + foreach ($stages as &$stage) {write_header_cell($class, $stage);} + echo "\n"; + + // write rows + foreach ($states as &$state) { + + $link = "czartool.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $state; + + echo "\n"; + write_table_cell($class, '%s', $link, $state); + write_table_cell($class, '%s', "", getStateAndFaults($db, $label,"chipRun", $state, "chip")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $label,"camRun", $state, "cam")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $label,"fakeRun", $state, "fake")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $label,"warpRun", $state, "warp")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $label,"stackRun", $state, "stack")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $label,"diffRun", $state, "diff")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $label,"magicRun", $state, "magic")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $label,"magicDSRun", $state, "magicDS")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $label,"distRun", $state, "dist")); + echo "\n"; + } + + echo "
\n"; +} + +########################################################################### +# +# Creates table for all labels showing all stages for given 'state' +# +########################################################################### +function showAllLabels($pass, $proj, $db, $stdsLabels, $distLabels, $stages, $states, $selectedState) { + + echo "

Current stdscience labels for '$selectedState' (any faults are shown in parentheses)

"; + + // set up table columns + $class = "list"; + echo "\n"; + echo "\n"; + write_header_cell($class, "label"); + write_header_cell($class, "distributing?"); + // echo "\n"; + // echo "\n"; + + foreach ($stages as &$stage) {write_header_cell($class, $stage);} + //foreach ($stages as &$stage) { + + // $link = "czartool.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $stage; + // echo "\n"; + // } + + echo "\n"; + + // write rows + foreach ($stdsLabels as &$stdsLabel) { + + $distributing = false; + foreach ($distLabels as &$distLabel) { + + if ($stdsLabel == $distLabel) { $distributing = true; break;} + } + + + // create link to label summary page for each label + $link = "czartool.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel; + + echo "\n"; + write_table_cell($class, '%s', $link, $stdsLabel); + write_table_cell($class, '%s', "", $distributing ? "yes" : "NO"); + write_table_cell($class, '%s', "", getStateAndFaults($db, $stdsLabel,"chipRun", $selectedState, "chip")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $stdsLabel,"camRun", $selectedState, "cam")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $stdsLabel,"fakeRun", $selectedState, "fake")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $stdsLabel,"warpRun", $selectedState, "warp")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $stdsLabel,"stackRun", $selectedState, "stack")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $stdsLabel,"diffRun", $selectedState, "diff")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $stdsLabel,"magicRun", $selectedState, "magic")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $stdsLabel,"magicDSRun", $selectedState, "magicDS")); + write_table_cell($class, '%s', "", getStateAndFaults($db, $stdsLabel,"distRun", $selectedState, "dist")); + echo "\n"; + } + + echo "
$myValue label$stage
\n"; +} + +########################################################################### +# +# Returns state and fault-count (if new) as a string +# +########################################################################### +function getStateAndFaults($db, $label, $table, $state, $stage) { + + $new = checkLabel($db, $label, $table, $state, $stage); + + if ($state != "new") {return $new;} + + $faults = countFaults($db, $label,$table,$stage); + + if ($faults < 1) {return $new;} + + return $new."(".$faults.")"; +} + +########################################################################### +# +# Returns count of exposures with this state for this label +# +########################################################################### +function checkLabel($db, $label, $table, $state, $stage) { + + $sql = "SELECT COUNT(state) FROM $table WHERE label LIKE '$label' AND state = '$state'"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + if (!$qry->fetchInto($row)) {echo "error with $sql
\n";} + return $row[0]; +} + +########################################################################### +# +# Returns count of faults for this stage and label +# +########################################################################### +function countFaults($db, $label, $table, $stage) { + + $joinTable = 0; + $id = $stage."_id"; + + if ($stage == "chip") {$joinTable="chipProcessedImfile";} + elseif ($stage == "cam") {$joinTable="camProcessedExp";} + elseif ($stage == "fake") {$joinTable="fakeProcessedImfile";} + elseif ($stage == "warp") {$joinTable="warpSkyfile";} + elseif ($stage == "stack") {$joinTable="stackSumSkyfile";} + elseif ($stage == "diff") {$joinTable="diffSkyfile";} + elseif ($stage == "magic") {$joinTable="magicNodeResult";} + elseif ($stage == "magicDS") {$id = "magic_ds_id"; $joinTable="magicDSFile";} + elseif ($stage == "dist") {$joinTable="distComponent";} + else {return -1;} + + $faultCol = $joinTable.".fault"; + + $sql = "SELECT COUNT(DISTINCT $id) FROM $table JOIN $joinTable USING ($id) WHERE label LIKE '$label' AND $faultCol != 0 AND $table.state = 'new'"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + if (!$qry->fetchInto($row)) {echo "error with $sql
\n";} + return $row[0]; +} +?> + Index: /branches/pap/ippMonitor/raw/ipp.imfiles.dat =================================================================== --- /branches/pap/ippMonitor/raw/ipp.imfiles.dat (revision 28002) +++ /branches/pap/ippMonitor/raw/ipp.imfiles.dat (revision 28003) @@ -21,2 +21,4 @@ menutop | menutop | link | simple plot - chip | simplePlot.php menutop | menutop | link | simple plot - cam | simplePlotcam.php +menutop | menutop | link | czartool | czartool.php + Index: /branches/pap/ippMonitor/scripts/checkServer.pl =================================================================== --- /branches/pap/ippMonitor/scripts/checkServer.pl (revision 28003) +++ /branches/pap/ippMonitor/scripts/checkServer.pl (revision 28003) @@ -0,0 +1,25 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; +use Getopt::Long; + +my $server = undef; +GetOptions ("server|s=s" => \$server); + +my @cmdOut = `echo "status ;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`; +my $line; +my $foundStatus = 0; +my $isRunning = 0; +foreach $line (@cmdOut) { + + chomp($line); + if ($line =~ m/Scheduler is stopped/) {$isRunning = 0; $foundStatus=1;last;} + if ($line =~ m/Scheduler is running/) {$isRunning = 1; $foundStatus=1;last;} + if ($line =~ m/Task Staus/) {last;} + +} + +($foundStatus) ? print "yes\n" : print "no\n"; +($isRunning) ? print "yes\n" : print "no\n"; + Index: /branches/pap/ippMonitor/scripts/getLabels.pl =================================================================== --- /branches/pap/ippMonitor/scripts/getLabels.pl (revision 28003) +++ /branches/pap/ippMonitor/scripts/getLabels.pl (revision 28003) @@ -0,0 +1,25 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; +use Getopt::Long; + +my $server = undef; +GetOptions ("server|s=s" => \$server); + +my @cmdOut = `echo "show.labels;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`; +my $line; +my $passedHeader=0; +foreach $line (@cmdOut) { + + chomp($line); + if ($line =~ m/pantasks:\s+(.*)/) { + + # HACK to get around 'dummy' label in stdscience + if ($1 !~ m/.*dummy.*/) {print "$1\n"}; + $passedHeader=1; + next; + } + + if ($passedHeader){print "$line\n";} +} Index: /branches/pap/ippMonitor/scripts/getServerStatus.pl =================================================================== --- /branches/pap/ippMonitor/scripts/getServerStatus.pl (revision 28003) +++ /branches/pap/ippMonitor/scripts/getServerStatus.pl (revision 28003) @@ -0,0 +1,18 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; +use Getopt::Long; + +my $server = undef; +GetOptions ("server|s=s" => \$server); + +my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`; +my $line; +my $passedHeader=0; +foreach $line (@cmdOut) { + + chomp($line); + if ($line =~ m/.*Task Status.*/) {$passedHeader=1;next;} + if ($passedHeader){print "$line
";} +} Index: /branches/pap/ippScripts/scripts/addstar_run.pl =================================================================== --- /branches/pap/ippScripts/scripts/addstar_run.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/addstar_run.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: /branches/pap/ippScripts/scripts/automate_stacks.pl =================================================================== --- /branches/pap/ippScripts/scripts/automate_stacks.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/automate_stacks.pl (revision 28003) @@ -56,5 +56,5 @@ # Grab options my ( $date, $camera, $dbname, $logfile, $verbose, $manual); -my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only); +my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $check_mode); my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old); @@ -67,6 +67,8 @@ 'logfile=s' => \$logfile, 'verbose' => \$verbose, + 'debug' => \$debug, 'isburning' => \$isburning, 'force_stack_count' => \$force_stack_count, + 'check' => \$check_mode, 'test_mode' => \$test_mode, 'this_target_only=s' => \$this_target_only, @@ -115,5 +117,5 @@ defined $queue_chips or defined $queue_stacks or defined $check_chips or defined $check_stacks or - defined $test_mode or defined $clean_old; + defined $test_mode or defined $clean_old or defined $check_mode; # Configurable parameters from our config file. @@ -126,6 +128,7 @@ my %stackable_list = (); my %reduction_class = (); -my $retention_time = 9000; - +my %clean_commands = (); +my %clean_retention = (); +my %noclean_list = (); # Grab the configuration data. my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; @@ -140,7 +143,22 @@ my $metadata = $mdcParser->parse(join "", @$stdout_buf); foreach my $entry (@{ $metadata }) { - if (${ $entry }{name} eq 'RETENTION_TIME') { - $retention_time = ${ $entry }{value}; - } + if (@{ $entry }{name} eq 'CLEAN_MODES') { + my @mode_data = @{ ${ $entry }{value} }; + my $this_mode = ''; + foreach my $mentry (@mode_data) { + if (${ $mentry }{name} eq 'MODE') { + $this_mode = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'COMMAND') { + $clean_commands{$this_mode} = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'RETENTION_TIME') { + $clean_retention{$this_mode} = ${ $mentry }{value}; + } + } + } +# if (${ $entry }{name} eq 'RETENTION_TIME') { +# $retention_time = ${ $entry }{value}; +# } elsif (${ $entry }{name} eq 'FILTERS') { push @filter_list, ${ $entry }{value}; @@ -172,4 +190,7 @@ $reduction_class{$this_target} = ${ $tentry }{value}; } + elsif (${ $tentry }{name} eq 'NOCLEAN') { + $noclean_list{$this_target} = ${ $tentry }{value}; + } } } @@ -217,11 +238,22 @@ if (defined($test_mode)) { $debug = 1; -} - -if (defined($check_registration) || defined($test_mode)) { + my $z; + foreach $z (@target_list) { + print "TARGET: $z $tessID_list{$z} $obsmode_list{$z} $object_list{$z} $comment_list{$z} $stackable_list{$z} $reduction_class{$z}\n"; + } + foreach $z (@filter_list) { + print "FILTER: $z\n"; + } + foreach $z (keys (%clean_commands)) { + print "CLEAN: $z $clean_commands{$z} $clean_retention{$z}\n"; + } + +} + +if (defined($check_registration) || defined($test_mode) || defined($check_mode)) { $metadata_out{nsState} = 'NEW'; my ($Nsummit_exp,$Nfaults) = check_summit_copy($date); return_metadata($date); - unless (defined($test_mode)) { exit(0); } + unless (defined($test_mode) || defined($check_mode)) { exit(0); } } if (defined($define_burntool) || defined($test_mode)) { @@ -236,5 +268,5 @@ unless (defined($test_mode)) { exit(0); } } -if (defined($check_chips) || defined($test_mode)) { +if (defined($check_chips) || defined($test_mode) || defined($check_mode)) { $metadata_out{nsState} = 'QUEUECHIPS'; &execute_chips($date,"pretend"); @@ -243,5 +275,5 @@ } return_metadata($date); - unless (defined($test_mode)) { exit(0); } + unless (defined($test_mode) || defined($check_mode)) { exit(0); } } if (defined($queue_chips)) { @@ -251,5 +283,5 @@ exit(0); } -if (defined($check_stacks) || defined($test_mode)) { +if (defined($check_stacks) || defined($test_mode) || defined($check_mode)) { $metadata_out{nsState} = 'TOWARP'; &execute_stacks($date,"pretend"); @@ -258,9 +290,12 @@ } return_metadata($date); - unless (defined($test_mode)) { exit(0); } + unless (defined($test_mode) || defined($check_mode)) { exit(0); } } if (defined($queue_stacks)) { $metadata_out{nsState} = 'STACKING'; &execute_stacks($date); + if ($metadata_out{nsState} eq 'QUEUESTACKING') { + $metadata_out{nsState} = 'STACKING_POSSIBLE_ERROR'; + } return_metadata($date); exit(0); @@ -420,8 +455,8 @@ } - if ($#{ $stderr_buf } > -1) { - $metadata_out{nsState} = 'ERROR'; - return(1); - } +# if ($#{ $stderr_buf } > -1) { +# $metadata_out{nsState} = 'ERROR'; +# return(1); +# } my $N = 0; @@ -429,6 +464,9 @@ foreach my $row (@burntool_entries) { my ($trash,$start,$end); - ($trash,$trash,$start,$end) = (split /\s+/,$row); - + (undef,$trash,$start,$end) = (split /\s+/,$row); + if ($trash ne 'burntool') { + $metadata_out{nsState} = 'ERROR'; + return(1); + } for (my $class_counter = 0; $class_counter < 60; $class_counter++) { $metadata_out{"bt${N}Begin"} = $start; @@ -650,5 +688,5 @@ my @input_exposures = split /\n/, (join '', @$stdout_buf); foreach my $entry (@input_exposures) { - my ($skycell, @trash) = split /\s+/, $entry; + my ($warp_tess_id,$skycell, @trash) = split /\s+/, $entry; @trash = verify_uniqueness_stack($skycell,$date,$target,$filter); $Nalready += $trash[0]; @@ -681,21 +719,32 @@ my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter); if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :( - # print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n"; + if ($debug == 1) { + print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n"; + } $metadata_out{nsState} = 'FORCETOWARP'; next; } if ($Nexposures == 0) { - # print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; + if ($debug == 1) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; + } next; } if ($Nalready != 0) { - # print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n"; + if ($debug == 1) { + print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n"; + } + unless ($metadata_out{nsState} eq 'FORCETOWARP') { + $metadata_out{nsState} = 'STACKING'; + } next; } - unless ($metadata_out{nsState} eq 'FORCETOWARP') { + unless (($metadata_out{nsState} eq 'FORCETOWARP')||($metadata_out{nsState} eq 'STACKING')) { $metadata_out{nsState} = 'QUEUESTACKS'; } unless(defined($pretend)) { - # print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n"; + if ($debug == 1) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n"; + } stack_queue($date,$target,$filter); } @@ -716,4 +765,8 @@ my $date = shift; my $target = shift; + my $mode = shift; + + my $command = $clean_commands{$mode}; + my $retention_time = $clean_retention{$mode}; my ($year,$month,$day) = split /-/,$date; @@ -725,6 +778,6 @@ my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target); - - my $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -data_group $data_group "; + my $args = $command; + $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; if ($debug == 1) { $args .= ' -pretend '; @@ -737,29 +790,22 @@ my $pretend = shift; - foreach my $target (@target_list) { - my ($cleaning_date,$args) = construct_cleantool_args($date,$target); - - my $command = $chiptool . $args; - print STDERR "$command\n"; - if (!(defined($pretend) || $debug == 1)) { + foreach my $mode (keys (%clean_commands)) { + foreach my $target (@target_list) { + if (exists($noclean_list{$target})) { + next; + } + my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); + + print STDERR "$command\n"; + if (!(defined($pretend) || $debug == 1)) { # print STDERR "BEAR IS DRIVING!?\n"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $command, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - } - $command = $warptool . $args; - print STDERR "$command\n"; - if (!(defined($pretend) || $debug == 1)) { -# print STDERR "BEAR IS DRIVING $pretend $debug!?\n"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $command, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + } + } } return(0); Index: /branches/pap/ippScripts/scripts/calibrate_dvo.pl =================================================================== --- /branches/pap/ippScripts/scripts/calibrate_dvo.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/calibrate_dvo.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/pap/ippScripts/scripts/camera_exp.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/camera_exp.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: /branches/pap/ippScripts/scripts/chip_imfile.pl =================================================================== --- /branches/pap/ippScripts/scripts/chip_imfile.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/chip_imfile.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: /branches/pap/ippScripts/scripts/detrend_correct_imfile.pl =================================================================== --- /branches/pap/ippScripts/scripts/detrend_correct_imfile.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/detrend_correct_imfile.pl (revision 28003) @@ -8,7 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; -print "command: @ARGV\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/detrend_norm_apply.pl =================================================================== --- /branches/pap/ippScripts/scripts/detrend_norm_apply.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/detrend_norm_apply.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/detrend_norm_calc.pl =================================================================== --- /branches/pap/ippScripts/scripts/detrend_norm_calc.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/detrend_norm_calc.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -56,5 +57,5 @@ # force det_type to be upper-case in this script -my $det_type = uc($det_type); +$det_type = uc($det_type); my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $det_id, $iter, $PS_EXIT_CONFIG_ERROR ); # IPP configuration Index: /branches/pap/ippScripts/scripts/detrend_norm_exp.pl =================================================================== --- /branches/pap/ippScripts/scripts/detrend_norm_exp.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/detrend_norm_exp.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/detrend_process_exp.pl =================================================================== --- /branches/pap/ippScripts/scripts/detrend_process_exp.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/detrend_process_exp.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/detrend_process_imfile.pl =================================================================== --- /branches/pap/ippScripts/scripts/detrend_process_imfile.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/detrend_process_imfile.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/detrend_reject_exp.pl =================================================================== --- /branches/pap/ippScripts/scripts/detrend_reject_exp.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/detrend_reject_exp.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/detrend_resid_imfile.pl =================================================================== --- /branches/pap/ippScripts/scripts/detrend_resid_imfile.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/detrend_resid_imfile.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/detrend_stack.pl =================================================================== --- /branches/pap/ippScripts/scripts/detrend_stack.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/detrend_stack.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/diff_skycell.pl =================================================================== --- /branches/pap/ippScripts/scripts/diff_skycell.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/diff_skycell.pl (revision 28003) @@ -7,6 +7,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; @@ -398,8 +399,8 @@ $command .= " -hostname $host" if defined $host; $command .= " -path_base $outroot" if defined $outroot; - $command .= " -dbname $dbname" if defined $dbname; } else { $command .= " -updatediffskyfile"; } + $command .= " -dbname $dbname" if defined $dbname; run(command => $command, verbose => $verbose); } Index: /branches/pap/ippScripts/scripts/dist_advancerun.pl =================================================================== --- /branches/pap/ippScripts/scripts/dist_advancerun.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/dist_advancerun.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -174,10 +175,10 @@ } &my_die("unable to find path", $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path; - my $component_dir = find_componentdir($destdir, $path); + my $component_dir = find_componentdir($destdir, $path); # print MANIFEST "$component METADATA\n"; print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n"; } print MANIFEST "END\n\n"; - close MANIFEST or + close MANIFEST or &my_die("Unable to close dirinfo file $dirinfo", $dist_id, $PS_EXIT_UNKNOWN_ERROR); } Index: /branches/pap/ippScripts/scripts/dist_bundle.pl =================================================================== --- /branches/pap/ippScripts/scripts/dist_bundle.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/dist_bundle.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; # print "\n\n"; -# print "Starting script $0 on $host\n\n"; +# print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -115,5 +116,5 @@ # Get the list of data products for this component -# note: We my_die in get_file_list if something goes wrong. +# note: We my_die in get_file_list if something goes wrong. my $file_list = get_file_list($stage, $component, $path_base, $clean); @@ -193,5 +194,5 @@ $num_files++; if ($image_type && $nan_masked_pixels) { - # save the + # save the if ($image_type eq 'image') { $image = $file_name; @@ -222,5 +223,5 @@ # One last check as to whether magic has been applied to the inputs - # Note: the sql for disttool -pendingcomponent won't select a component that + # Note: the sql for disttool -pendingcomponent won't select a component that # requires magic and hasn't been magicked, but we check again here @@ -327,5 +328,5 @@ ### Pau. -# return the image type (image, mask, or variance) if this file rule refers to +# return the image type (image, mask, or variance) if this file rule refers to # one of the big fits files that are not included in a clean distribution sub get_image_type { @@ -360,5 +361,5 @@ sub open_with_retries { my $name = shift; - + my $tries = 1; my $max_tries = 5; @@ -372,5 +373,5 @@ } } - + &my_die("failed to open $name after $max_tries tries\n", $component, $PS_EXIT_DATA_ERROR) if (!$opened); Index: /branches/pap/ippScripts/scripts/dist_cleanup.pl =================================================================== --- /branches/pap/ippScripts/scripts/dist_cleanup.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/dist_cleanup.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/dist_component.pl =================================================================== --- /branches/pap/ippScripts/scripts/dist_component.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/dist_component.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/dist_defineruns.pl =================================================================== --- /branches/pap/ippScripts/scripts/dist_defineruns.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/dist_defineruns.pl (revision 28003) @@ -11,6 +11,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/dist_make_fileset.pl =================================================================== --- /branches/pap/ippScripts/scripts/dist_make_fileset.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/dist_make_fileset.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -46,5 +47,5 @@ 'dist_id=s' => \$dist_id, # distribution run identifier 'dist_dir=s' => \$dist_dir, # directory containing dist run outputs - 'target_id=s' => \$target_id, # + 'target_id=s' => \$target_id, # 'stage=s' => \$stage, # raw, chip, camera, fake, warp, stack, or diff 'stage_id=s' => \$stage_id, # exp_id, chip_id, etc. @@ -209,5 +210,5 @@ my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; - my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) or die "Cannot connect to database.\n"; @@ -227,10 +228,10 @@ # - # we are a long ways away from the rawExp in the pipeline. Rather than do some + # we are a long ways away from the rawExp in the pipeline. Rather than do some # very long joins in disttool, we look up the exp_name in the database using DBI # my $dbh = getDBHandle($ipprc, $dbname); - my $query; + my $query; if ($stage eq 'raw') { Index: /branches/pap/ippScripts/scripts/dqstats_bundle.pl =================================================================== --- /branches/pap/ippScripts/scripts/dqstats_bundle.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/dqstats_bundle.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; @@ -37,30 +38,30 @@ my ( $dqstats_id, $camera, $uri, $dbname, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( - 'dqstats_id=s' => \$dqstats_id, # dqstatsrun ID - 'camera|c=s' => \$camera, # camera - 'dbname|d=s' => \$dbname, # database name - 'uri=s' => \$uri, # output file destination - 'verbose' => \$verbose, # print to stdout - 'no-update' => \$no_update, # Update the database - 'no-op' => \$no_op, # don't do the operations - 'redirect-output' => \$redirect, - 'save-temps' => \$save_temps, # save temporary files - ) or pod2usage( 2 ); + 'dqstats_id=s' => \$dqstats_id, # dqstatsrun ID + 'camera|c=s' => \$camera, # camera + 'dbname|d=s' => \$dbname, # database name + 'uri=s' => \$uri, # output file destination + 'verbose' => \$verbose, # print to stdout + 'no-update' => \$no_update, # Update the database + 'no-op' => \$no_op, # don't do the operations + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # save temporary files + ) or pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( -msg => "Required options --dqstats_id --camera", - -exitval => 3, - ) unless + -exitval => 3, + ) unless defined $dqstats_id and defined $camera; -my $ipprc = PS::IPP::Config->new( $camera ) +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $dqstats_id, $PS_EXIT_CONFIG_ERROR ); # IPP config -#my $logDest = $ipprc->filename("LOG.EXP", $uri) +#my $logDest = $ipprc->filename("LOG.EXP", $uri) # or my_die("Missing entry from camera config", $dqstats_id, $PS_EXIT_CONFIG_ERROR); if ($redirect) { -# $ipprc->redirect_output($logDest) -# or my_die( "Unable to redirect output", $dqstats_id, $PS_EXIT_SYS_ERROR ); +# $ipprc->redirect_output($logDest) +# or my_die( "Unable to redirect output", $dqstats_id, $PS_EXIT_SYS_ERROR ); print "\n\n"; print "Starting script $0 on $host\n\n"; @@ -84,16 +85,16 @@ unless ($no_op) { # This bit needs to make the bundle. - + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $bundle_command, verbose => $verbose); + run(command => $bundle_command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to create the bundle.: $error_code", $dqstats_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to create the bundle.: $error_code", $dqstats_id, $error_code); } - + # Parse stdout_buf to get output file name ## Begin partial my ($tempFile, $tempName) = tempfile( "/tmp/dqstats.XXXX", - UNLINK => !$save_temps, SUFFIX => 'dsin' ); + UNLINK => !$save_temps, SUFFIX => 'dsin' ); print $tempFile $uri . '|||notset' . "\n"; @@ -102,10 +103,10 @@ # It can't be that simple, can it? This should probably add a fault on failure. ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $register_cmd, verbose => $verbose); + run(command => $register_cmd, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to register the bundle.: $error_code", $dqstats_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to register the bundle.: $error_code", $dqstats_id, $error_code); } - + # We no longer need the file generated by dqstatstool. unlink($uri); @@ -118,9 +119,9 @@ # This bit needs to set the database state to full my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $update_command, verbose => $verbose); + run(command => $update_command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to add result to database: $error_code\n"); - exit($error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); } } else { @@ -139,5 +140,5 @@ carp($msg); if (defined $dqstats_id and not $no_update) { - # There is no fault handling, which may be a mistake. This is where a fault coding would be. + # There is no fault handling, which may be a mistake. This is where a fault coding would be. # my $command = "$addtool -add_id $add_id"; # $command .= " -addprocessedexp"; Index: /branches/pap/ippScripts/scripts/fake_imfile.pl =================================================================== --- /branches/pap/ippScripts/scripts/fake_imfile.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/fake_imfile.pl (revision 28003) @@ -24,6 +24,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: /branches/pap/ippScripts/scripts/flatcorr_init.pl =================================================================== --- /branches/pap/ippScripts/scripts/flatcorr_init.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/flatcorr_init.pl (revision 28003) @@ -1,5 +1,5 @@ #!/usr/bin/env perl -## USAGE:flatcorr_init.pl +## USAGE:flatcorr_init.pl ## given dbname, dvodb, filter, time range, camera, telescope, etc? ## select matching images and register as a new flatcorr run @@ -24,6 +24,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -68,7 +69,7 @@ my $caltool = can_run('caltool') or (warn "Can't find caltool" and $missing_tools = 1); -if ($missing_tools) { +if ($missing_tools) { warn ("Can't find required tools"); - exit($PS_EXIT_CONFIG_ERROR); + exit($PS_EXIT_CONFIG_ERROR); } @@ -90,5 +91,5 @@ cache_run(command => $command, verbose => 1); - unless ($success) { + unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RESORT", $status, $dbname); @@ -99,15 +100,15 @@ { foreach my $filter (@filters) { - my $command = "$relphot $filter"; - $command .= "-D CATDIR $catdir"; - $command .= "-region $RAs $RAe $DECs $DECe"; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - cache_run(command => $command, verbose => 1); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); - } + my $command = "$relphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } } } @@ -117,15 +118,15 @@ if (0) { foreach my $filter (@filters) { - my $command = "$uniphot $filter"; - $command .= "-D CATDIR $catdir"; - $command .= "-region $RAs $RAe $DECs $DECe"; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - cache_run(command => $command, verbose => 1); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "UNIPHOT", $status, $dbname); - } + my $command = "$uniphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "UNIPHOT", $status, $dbname); + } } } @@ -137,9 +138,9 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - cache_run(command => $command, verbose => 1); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); } } @@ -151,9 +152,9 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - cache_run(command => $command, verbose => 1); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); } } @@ -185,14 +186,14 @@ my $region = shift; my $last_step = shift; - my $status = shift; - my $dbname = shift; + my $status = shift; + my $dbname = shift; carp($msg); if (defined $dvo_id && defined $region && defined $last_step && defined $status and not $no_update) { my $command = "$caltool -addcalrun"; - $command .= " -dvo_id $dvo_id"; + $command .= " -dvo_id $dvo_id"; $command .= " -region $region"; - $command .= " -last_step $last_step"; - $command .= " -status $status"; + $command .= " -last_step $last_step"; + $command .= " -status $status"; $command .= " -dbname $dbname" if defined $dbname; system ($command); Index: /branches/pap/ippScripts/scripts/flatcorr_proc.pl =================================================================== --- /branches/pap/ippScripts/scripts/flatcorr_proc.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/flatcorr_proc.pl (revision 28003) @@ -11,7 +11,7 @@ # dvoMakeCorr -file outgrid.fits -ref ref.fits outcorr -# dettool -register -det_type FLATCORR -filelevel (level) -workdir -inst, etc - -# foreach $imfile () +# dettool -register -det_type FLATCORR -filelevel (level) -workdir -inst, etc + +# foreach $imfile () # dettool -register_imfile -uri, etc, etc @@ -23,6 +23,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -74,11 +75,11 @@ my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); my $dvoMakeCorr = can_run('dvoMakeCorr') or (warn "Can't find dvoMakeCorr" and $missing_tools = 1); -my $detselect = can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); -my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); -my $flatcorr = can_run('flatcorr') or (warn "Can't find flatcorr" and $missing_tools = 1); - -if ($missing_tools) { +my $detselect = can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $flatcorr = can_run('flatcorr') or (warn "Can't find flatcorr" and $missing_tools = 1); + +if ($missing_tools) { warn ("Can't find required tools"); - exit($PS_EXIT_CONFIG_ERROR); + exit($PS_EXIT_CONFIG_ERROR); } @@ -95,5 +96,5 @@ $ipprc->outroot_prepare($outcorr); -if (not -e "$dvodb/flatcorr") { +if (not -e "$dvodb/flatcorr") { mkdir "$dvodb/flatcorr" or &my_die ("Unable to make output directory for relphot $dvodb/flatcorr", $corr_id, 3); } @@ -104,17 +105,17 @@ my ($DECs, $DECe) = split (",", $coords[1]); -# Run addstar -resort to ensure the db is indexed +# Run addstar -resort to ensure the db is indexed # XXX addstar should be able to recognize and skip indexed tables { - my $command = "$addstar -resort"; + my $command = "$addstar -resort"; $command .= " -D CATDIR $dvodb"; $command .= " -region $RAs $RAe $DECs $DECe"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform addstar -resort for dvodb $dvodb on region $region: $error_code", $corr_id, $error_code); + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort for dvodb $dvodb on region $region: $error_code", $corr_id, $error_code); } } @@ -135,9 +136,9 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform relphot -grid for dvodb $dvodb on region $region: $error_code", $corr_id, $error_code); + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform relphot -grid for dvodb $dvodb on region $region: $error_code", $corr_id, $error_code); } } @@ -176,5 +177,5 @@ # Push the results into the database -{ +{ my $command = "$flatcorr -addprocess"; $command .= " -corr_id $corr_id"; @@ -184,13 +185,13 @@ unless ($no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn ("Unable to perform regtool -addprocessedimfile: $error_code"); - exit($error_code); - } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } } else { - print "skipping command: $command\n"; + print "skipping command: $command\n"; } } @@ -218,5 +219,5 @@ &my_die("Unable to parse metadata list", $corr_id, $PS_EXIT_PROG_ERROR); - # check for existence + # check for existence my $file = $$files[0]; my $chip_id = $file->{chip_id}; @@ -228,5 +229,5 @@ # flatcorr -inputimfile -chip_id $chip_id sub get_imfiles { - my $chip_id = shift; + my $chip_id = shift; my $command = "$flatcorr -inputimfile"; @@ -273,9 +274,9 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to register new detrend: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to register new detrend: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); } @@ -298,39 +299,39 @@ foreach my $file (@$files) { - # create the detrend correction for the imfiles based on the input imfiles - my $reffile = $file->{uri}; - my $class_id = $file->{class_id}; - - my $uri = $ipprc->filename("DVOCORR.OUTPUT", $outcorr, $class_id); - unless ($uri) { - &my_die ("Unable to find DVOCORR.OUTPUT in filerules", $corr_id, $PS_EXIT_PROG_ERROR); - } - - my $command = "$dvoMakeCorr $outcorr"; - $command .= " -file $outgrid.fits"; - $command .= " -ref $reffile"; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform dvoMakeCorr: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); - } - - # register the detrend correction imfile - $command = "$dettool -register_detrend_imfile"; - $command .= " -det_id $det_id"; - $command .= " -class_id $class_id"; - $command .= " -uri $uri"; - $command .= " -dbname $dbname" if defined $dbname; - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to register new detrend: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); - } + # create the detrend correction for the imfiles based on the input imfiles + my $reffile = $file->{uri}; + my $class_id = $file->{class_id}; + + my $uri = $ipprc->filename("DVOCORR.OUTPUT", $outcorr, $class_id); + unless ($uri) { + &my_die ("Unable to find DVOCORR.OUTPUT in filerules", $corr_id, $PS_EXIT_PROG_ERROR); + } + + my $command = "$dvoMakeCorr $outcorr"; + $command .= " -file $outgrid.fits"; + $command .= " -ref $reffile"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform dvoMakeCorr: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); + } + + # register the detrend correction imfile + $command = "$dettool -register_detrend_imfile"; + $command .= " -det_id $det_id"; + $command .= " -class_id $class_id"; + $command .= " -uri $uri"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to register new detrend: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); + } } } @@ -345,5 +346,5 @@ if (not $no_update) { my $command = "$flatcorr -addprocess"; - $command .= " -corr_id $corr_id"; + $command .= " -corr_id $corr_id"; $command .= " -fault $exit_code"; $command .= " -hostname $host" if defined $host; Index: /branches/pap/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- /branches/pap/ippScripts/scripts/ipp_cleanup.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/ipp_cleanup.pl (revision 28003) @@ -14,5 +14,4 @@ use File::Spec; use PS::IPP::Config 1.01 qw( :standard ); -use PS::IPP::Metadata::List qw( parse_md_list ); use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); use Pod::Usage qw( pod2usage ); @@ -108,10 +107,7 @@ } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR); - - # extract the metadata for the files into a hash list - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "chip", $stage_id, $PS_EXIT_PROG_ERROR); # loop over all of the imfiles, determine the path_base and class_id for each @@ -196,5 +192,5 @@ # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -249,9 +245,6 @@ &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code); } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "camera", $stage_id, $PS_EXIT_PROG_ERROR); - - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "camera", $stage_id, $PS_EXIT_PROG_ERROR); my $n_exps = @$exps; @@ -316,5 +309,5 @@ # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -375,9 +368,6 @@ exit(0); } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $skyfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR); - - $skyfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "warp", $stage_id, $PS_EXIT_PROG_ERROR); my @files = (); @@ -457,5 +447,5 @@ # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -515,9 +505,6 @@ } - my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + $skyfiles = $mdcParser->parse_list(join "", @{ $stdout_buf }) or &my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR); - - $skyfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "stack", $stage_id, $PS_EXIT_PROG_ERROR); my @files = (); @@ -594,5 +581,5 @@ # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -652,9 +639,6 @@ } - my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + $skyfiles = $mdcParser->parse_list(join "", @{ $stdout_buf }) or &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR); - - $skyfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "diff", $stage_id, $PS_EXIT_PROG_ERROR); my @files = (); @@ -747,5 +731,5 @@ # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -806,10 +790,7 @@ } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR); - - # extract the metadata for the files into a hash list - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR); # loop over all of the imfiles, determine the path_base and class_id for each @@ -927,10 +908,7 @@ - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - # extract the metadata for the files into a hash list - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); } # loop over all of the imfiles, determine the path_base and class_id for each @@ -1013,8 +991,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR); - my $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1068,10 +1044,7 @@ # if there are no detResidImfiles (@$stdout_buf == 0), then silently exit. if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - # extract the metadata for the files into a hash list - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); } # loop over all of the imfiles, determine the path_base and class_id for each @@ -1156,8 +1129,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR); - my $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1202,9 +1173,6 @@ &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); } - my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + $stacks = $mdcParser->parse_list(join "", @{ $stdout_buf }) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - $stacks = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); my @files = (); @@ -1286,8 +1254,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - my $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1327,9 +1293,6 @@ &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - my $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1380,8 +1343,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1425,9 +1386,6 @@ &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1511,8 +1469,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1559,9 +1515,6 @@ exit 0; } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1643,8 +1596,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { Index: /branches/pap/ippScripts/scripts/ipp_detrend_combine.pl =================================================================== --- /branches/pap/ippScripts/scripts/ipp_detrend_combine.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/ipp_detrend_combine.pl (revision 28003) @@ -7,6 +7,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -32,24 +33,24 @@ $workdir, $dbname, $no_update); GetOptions( - 'det_type=s' => \$det_type, # Detrend type for new detrend - 'filelevel=s' => \$filelevel, # File level for new detrend - 'inst=s' => \$inst, # Instrument for new detrend - 'telescope=s' => \$telescope, # Telescope for new detrend - 'filter=s' => \$filter, # Filter name for new detrend - 'det_id1=s' => \$det_id1, # Detrend id for detrend 1 - 'iteration1=s' => \$iter1, # Iteration for detrend 1 - 'det_id2=s' => \$det_id2, # Detrend id for detrend 2 - 'iteration2=s' => \$iter2, # Iteration for detrend 2 - 'operation=s' => \$operation, # Operation to perform on files - 'mask' => \$mask, # Operation is on a mask - 'workdir=s' => \$workdir, # Working directory for output files - 'dbname=s' => \$dbname, # Database name - 'no-update' => \$no_update, # Don't update the database - ) or pod2usage( 2 ); + 'det_type=s' => \$det_type, # Detrend type for new detrend + 'filelevel=s' => \$filelevel, # File level for new detrend + 'inst=s' => \$inst, # Instrument for new detrend + 'telescope=s' => \$telescope, # Telescope for new detrend + 'filter=s' => \$filter, # Filter name for new detrend + 'det_id1=s' => \$det_id1, # Detrend id for detrend 1 + 'iteration1=s' => \$iter1, # Iteration for detrend 1 + 'det_id2=s' => \$det_id2, # Detrend id for detrend 2 + 'iteration2=s' => \$iter2, # Iteration for detrend 2 + 'operation=s' => \$operation, # Operation to perform on files + 'mask' => \$mask, # Operation is on a mask + 'workdir=s' => \$workdir, # Working directory for output files + 'dbname=s' => \$dbname, # Database name + 'no-update' => \$no_update, # Don't update the database + ) or pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( -msg => "Required options --det_type --filelevel --inst --telescope --det_id1 --iteration1 --det_id2 --iteration2 --workdir", - -exitval => 3, - ) + -exitval => 3, + ) unless defined $det_type and defined $filelevel @@ -65,6 +66,6 @@ $ipprc->define_camera($inst); -my $STATS = - [ +my $STATS = + [ # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, @@ -90,15 +91,15 @@ die("File lists for detrends have differing lengths") unless scalar keys %$files1 == scalar keys %$files2; -my ($det_id, $iter); # Detrend identifier for the new detrend +my ($det_id, $iter); # Detrend identifier for the new detrend unless ($no_update) { my $command = "$dettool -register_detrend -det_type $det_type -filelevel $filelevel -workdir $workdir " . - "-inst $inst -telescope $telescope"; # Command to run + "-inst $inst -telescope $telescope"; # Command to run $command .= " -filter $filter" if defined $filter; $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); + run(command => $command, verbose => 1); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - die("Unable to run dettool -register_detrend: $error_code"); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend: $error_code"); } @@ -137,8 +138,8 @@ $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); + run(command => $command, verbose => 1); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - die("Unable to run ppArith: $error_code"); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run ppArith: $error_code"); } @@ -149,28 +150,28 @@ my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser { - my $statsFile; # File handle - open $statsFile, $ipprc->file_resolve($outStats) or die("Can't open stats file $outStats: $!"); - my @contents = <$statsFile>; # Contents of file - close $statsFile; - - my $metadata = $mdcParser->parse(join "", @contents) or die("Unable to parse metadata config doc"); - - unless ($stats->parse($metadata)) { - &my_die("Failure extracting metadata from the statistics output file.\n"); - } + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outStats) or die("Can't open stats file $outStats: $!"); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + my $metadata = $mdcParser->parse(join "", @contents) or die("Unable to parse metadata config doc"); + + unless ($stats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n"); + } } # Register the imfile unless ($no_update) { - my $command = "$dettool -register_detrend_imfile -det_id $det_id "; # Command to run - $command .= " -class_id $class_id -uri $outName -path_base $outRoot"; - $command .= $stats->cmdflags(); - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - die("Unable to run dettool -register_detrend_imfile: $error_code"); - } + my $command = "$dettool -register_detrend_imfile -det_id $det_id "; # Command to run + $command .= " -class_id $class_id -uri $outName -path_base $outRoot"; + $command .= $stats->cmdflags(); + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend_imfile: $error_code"); + } } } @@ -183,14 +184,14 @@ sub filelist { - my $det_id = shift; # Detrend identifier - my $iter = shift; # Iteration + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration my $command = "$detselect -select -det_id $det_id -iteration $iter"; # Command to run $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); + run(command => $command, verbose => 1); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - die("Unable to run detselect: $error_code"); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run detselect: $error_code"); } @@ -202,8 +203,8 @@ foreach my $item ( @$list ) { - my $class_id = $item->{class_id}; - die("Multiple definitions of class_id=$class_id found for det_id=$det_id, iteration=$iter\n") if - defined $files{$class_id}; - $files{$class_id} = parse_md_list($md); + my $class_id = $item->{class_id}; + die("Multiple definitions of class_id=$class_id found for det_id=$det_id, iteration=$iter\n") if + defined $files{$class_id}; + $files{$class_id} = parse_md_list($md); } Index: /branches/pap/ippScripts/scripts/lossy_compress_imfile.pl =================================================================== --- /branches/pap/ippScripts/scripts/lossy_compress_imfile.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/lossy_compress_imfile.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -86,36 +87,36 @@ if ($state eq 'goto_compressed') { # Find the actual filename for this run: - &my_die("Couldn't find input file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) - unless ($ipprc->file_exists($uri)); + &my_die("Couldn't find input file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) + unless ($ipprc->file_exists($uri)); my $uriReal = $ipprc->file_resolve( $uri ); # Create a compressed version: my $compUri = $uri . ".fz"; - &my_die("Output compressed file already exists: $compUri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) - if ($ipprc->file_exists($compUri)); + &my_die("Output compressed file already exists: $compUri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) + if ($ipprc->file_exists($compUri)); my $compReal= $ipprc->file_resolve( $compUri, 'create'); # Apparently we need to funpack before fpacking. Probably should have realized that beforehand. my $tempfile = new File::Temp ( TEMPLATE => "${exp_name}.XXXX", - DIR => '/tmp/', - UNLINK => !$save_temps, - SUFFIX => '.fits'); + DIR => '/tmp/', + UNLINK => !$save_temps, + SUFFIX => '.fits'); my $tempReal = $tempfile->filename; my $uncompress_command = "$funpack -S $uriReal > $tempReal"; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run(command => $uncompress_command, verbose => $verbose); + run(command => $uncompress_command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to uncompress file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to uncompress file: $uri -> $compUri: $error_code", + $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); } my $compress_command = "$fpack -h -s 8 -S $tempReal > $compReal"; print STDERR "$compReal $uriReal $compress_command\n"; ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run(command => $compress_command, verbose => $verbose); + run(command => $compress_command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to compress file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to compress file: $uri -> $compUri: $error_code", + $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); } my $database_command = "$regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state compressed"; @@ -123,17 +124,17 @@ &my_die("Expected output compressed file not found: $compUri\n", $exp_id,$exp_name,$class_id, $uri,$PS_EXIT_SYS_ERROR) - unless ($ipprc->file_exists($compUri)); + unless ($ipprc->file_exists($compUri)); if ($no_update || $no_op) { - print STDERR "NOUPDATE: $database_command\n"; + print STDERR "NOUPDATE: $database_command\n"; } else { - ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run(command => $database_command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to update database file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); - } + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $database_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to update database file: $uri -> $compUri: $error_code", + $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + } } exit(0); @@ -149,28 +150,28 @@ # Confirm we have both files &my_die("Couldn't find original file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) - unless($ipprc->file_exists($uri)); + unless($ipprc->file_exists($uri)); my $uriReal = $ipprc->file_resolve( $uri ); - + my $compUri = $uri . ".fz"; unless($ipprc->file_exists($compUri)) { - &my_die("Couldn't find compressed version of the file: $compUri\n", - $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); - - # If that die is removed, this will compress things as well. - # -# &my_die("Output compressed file already exists: $compUri\n", -# $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) -# if ($ipprc->file_exists($compUri)); -# my $compReal= $ipprc->file_resolve( $compUri, 'create'); - -# my $compress_command = "$fpack -h -s 8 -S $uriReal > $compReal"; -# print STDERR "$compReal $uriReal $compress_command\n"; -# my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = -# run(command => $compress_command, verbose => $verbose); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# &my_die("Unable to compress file: $uri -> $compUri: $error_code", -# $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); -# } + &my_die("Couldn't find compressed version of the file: $compUri\n", + $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + + # If that die is removed, this will compress things as well. + # +# &my_die("Output compressed file already exists: $compUri\n", +# $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) +# if ($ipprc->file_exists($compUri)); +# my $compReal= $ipprc->file_resolve( $compUri, 'create'); + +# my $compress_command = "$fpack -h -s 8 -S $uriReal > $compReal"; +# print STDERR "$compReal $uriReal $compress_command\n"; +# my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = +# run(command => $compress_command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to compress file: $uri -> $compUri: $error_code", +# $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); +# } } @@ -181,10 +182,10 @@ unless ($no_op) { - $neb->replicate($compUri); - $neb->swap($uri,$compUri) or - &my_die("Nebulous swap failed between $uri and $compUri", - $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); - } - # Update database + $neb->replicate($compUri); + $neb->swap($uri,$compUri) or + &my_die("Nebulous swap failed between $uri and $compUri", + $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + } + # Update database my $database_command = "$regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state lossy "; @@ -192,20 +193,20 @@ $database_command .= " -dbname $dbname" if defined $dbname; - + if ($no_update || $no_op) { - print STDERR "NOUPDATE: $database_command\n"; + print STDERR "NOUPDATE: $database_command\n"; } else { - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run(command => $database_command, verbose => $verbose); - unless($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - $neb->swap($uri,$compUri) or - &my_die("DB update failed and Nebulous swap-back failed between $uri and $compUri", - $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); - &my_die("Unable to update database file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); - } - # remove original version + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $database_command, verbose => $verbose); + unless($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + $neb->swap($uri,$compUri) or + &my_die("DB update failed and Nebulous swap-back failed between $uri and $compUri", + $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + &my_die("Unable to update database file: $uri -> $compUri: $error_code", + $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + } + # remove original version $neb->delete($compUri); } Index: /branches/pap/ippScripts/scripts/magic_destreak.pl =================================================================== --- /branches/pap/ippScripts/scripts/magic_destreak.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/magic_destreak.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -41,5 +42,5 @@ # Parse the command-line arguments my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base, $cam_reduction); -my ($outroot, $recoveryroot); +my ($outroot, $recoveryroot, $magicked); my ($replace, $release); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); @@ -61,4 +62,5 @@ 'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels 'replace=s' => \$replace, # replace the input images with the results. + 'magicked=s' => \$magicked, # magicked state of the run 'release' => \$release, # NAN masked pixels for release 'save-temps' => \$save_temps, # Save temporary files? @@ -71,5 +73,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --uri --path_base --outroot", +pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --uri --path_base --outroot --magicked", -exitval => 3) unless defined $magic_ds_id and @@ -82,5 +84,6 @@ defined $uri and defined $path_base and - defined $outroot; + defined $outroot and + defined $magicked; my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -209,5 +212,4 @@ &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); - $temp_dir = tempdir( CLEANUP => !$save_temps); ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps); @@ -218,4 +220,7 @@ } else { # diff run must have been cleaned up, need to create this skycell file on the fly + if (!defined $temp_dir ) { + $temp_dir = tempdir( CLEANUP => !$save_temps); + } my $skycell_id = $skycell->{skycell_id}; $skycell_uri = "$temp_dir/$skycell_id"; @@ -247,5 +252,6 @@ my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + # note verbose == 0 to avoid polluting log files with almost always useless information + run(command => $command, verbose => 0); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -266,5 +272,7 @@ $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base); - if ($dynamicMasks) { + # if magicked is non-zero we are updating a previously magicked component. In this case we don't + # touch the camera mask + if (!$magicked and $dynamicMasks) { $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id); $ch_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id); Index: /branches/pap/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- /branches/pap/ippScripts/scripts/magic_destreak_cleanup.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/magic_destreak_cleanup.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -62,13 +63,4 @@ $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile; -if (0) { -my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' ); -&my_die("cannot find NEB_SERVER in site configuration", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer; - -my $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); }; -if ($@ or not defined $nebulous) { - &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); -} -} $dbname = metadataLookupStr( $ipprc->{_siteConfig}, 'DBNAME' ) if !$dbname; @@ -107,4 +99,7 @@ +&my_die("cleanup not supported for camera stage", $magic_ds_id, $PS_EXIT_PROG_ERROR) if $stage eq "camera"; + + &my_die("unexpected run state found: $state", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $state ne "goto_cleaned"; &my_die("clean not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw"; @@ -129,4 +124,8 @@ $dbh->disconnect() or warn $dbh->errstr; +# We no longer clean up the camera stage mask files, but the code was left in place in case +# we change our minds. +my $cleanup_cam_mask = 0; + my $dynamicMasks; # Use dynamic masks? foreach my $comp (@components) { @@ -139,5 +138,5 @@ if ($stage eq "chip") { # Check to see if we're using dynamic masks - if (!defined $dynamicMasks) { + if ($cleanup_cam_mask && !defined $dynamicMasks) { # Get the PSASTRO recipe my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; Index: /branches/pap/ippScripts/scripts/magic_destreak_defineruns.pl =================================================================== --- /branches/pap/ippScripts/scripts/magic_destreak_defineruns.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/magic_destreak_defineruns.pl (revision 28003) @@ -11,6 +11,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- /branches/pap/ippScripts/scripts/magic_destreak_revert.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/magic_destreak_revert.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -37,5 +38,5 @@ # Parse the command-line arguments -my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction); +my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction, $magicked); my ($outroot, $recoveryroot, $replace, $release, $bytes, $md5sum); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); @@ -54,4 +55,5 @@ 'replace=s' => \$replace, # replace the input images with the results. 'bothways=s' => \$bothways, # run has inverse files (bothways diff) + 'magicked=s' => \$magicked, # magicked state of the run 'save-temps' => \$save_temps, # Save temporary files? 'dbname=s' => \$dbname, # Database name @@ -63,5 +65,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --magic_ds_id --camera --stage --stage_id --component --path_base --outroot", +pod2usage( -msg => "Required options: --magic_ds_id --camera --stage --stage_id --component --path_base --outroot --magicked", -exitval => 3) unless defined $magic_ds_id and @@ -71,4 +73,5 @@ defined $component and defined $path_base and + defined $magicked and defined $outroot; @@ -76,4 +79,6 @@ &my_die("cam_path_base is required for chip stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($stage eq 'chip' and !$cam_path_base); + + my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -168,5 +173,6 @@ my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + # note verbose == 0 to keep from polluting log file with lots of usually useless information + run(command => $command, verbose => 0); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -181,10 +187,12 @@ } + $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id); + $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id); + # we use the mask output from the camera stage for input and replace # the output of the chip stage with that mask as well. - $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id); - $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id); - - if ($dynamicMasks) { + # Note that when destreaking as part of an update $magicked is non-zero. + # In this case we do not touch the camera stage mask so there is no need to revert it + if (!$magicked and $dynamicMasks) { $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id); $ch_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id); @@ -194,5 +202,5 @@ $bimage = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $class_id); - # This is kludgey but correct + # This is somewhat kludgey but it works whether the mask is camera mask or chip mask $bmask = dirname($backup_path_base) . "/SR_" . basename($mask); $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id); Index: /branches/pap/ippScripts/scripts/magic_process.pl =================================================================== --- /branches/pap/ippScripts/scripts/magic_process.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/magic_process.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -98,13 +99,11 @@ } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $inputs = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $magic_id, $node, $PS_EXIT_PROG_ERROR); - - $inputs = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $magic_id, $node, $PS_EXIT_PROG_ERROR); } my @outputs; +my $inverse; # Using inverse diff? ### Do the heavy lifting { @@ -262,4 +261,8 @@ cat_list_to_list($mfh, $in_path_base, "mask.list"); cat_list_to_list($wfh, $in_path_base, "weight.list"); + + if ($innode->{inverse}) { + $inverse = 1; + } } close $in_fh; @@ -329,4 +332,5 @@ } } + if ($node eq "root") { my $streaks_file = "$outroot.streaks"; @@ -347,21 +351,56 @@ &run_verifystreaks($baseroot); - my $command = "$magictool -addmask"; - $command .= " -magic_id $magic_id"; - $command .= " -uri $streaks_file"; - $command .= " -streaks $num_streaks"; - $command .= " -dbname $dbname" if defined $dbname; - - # Add the processed file to the database - unless ($no_update) { + my $exp_id; # Exposure identifier + my $cam_path; # Camera stage path_base + { + my $command = "magictool -exposure -magic_id $magic_id"; + $command .= " -inverse" if defined $inverse; + $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - # This isn't going to work because our result was already entered. - &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code); - } - } else { - print "Skipping command: $command\n"; + my $exposures = $mdcParser->parse_list(join "", @$stdout_buf); + my $exp = $$exposures[0]; # Exposure of interest (should only be one) + + $exp_id = $exp->{exp_id}; + $cam_path = $exp->{path_base}; + } + + { + my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path); # Astrometry file + my $streaks = "$outroot.streaks"; # Streaks file + my $clusters = "$baseroot.verify/${exp_id}_clusterPos.txt"; # Clusters file + + my $command = "ppCoord -astrom $astrom -streaks $streaks"; + if ($ipprc->file_exists($clusters)) { + $command .= " -clusters $clusters"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + open my $coords, "> $outroot.coords"; + print $coords join("", @$stdout_buf); + close $coords; + } + + + { + my $command = "$magictool -addmask"; + $command .= " -magic_id $magic_id"; + $command .= " -uri $streaks_file"; + $command .= " -streaks $num_streaks"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + # This isn't going to work because our result was already entered. + &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code); + } + } else { + print "Skipping command: $command\n"; + } } } Index: /branches/pap/ippScripts/scripts/magic_tree.pl =================================================================== --- /branches/pap/ippScripts/scripts/magic_tree.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/magic_tree.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/make_burntool_pcontrol.pl =================================================================== --- /branches/pap/ippScripts/scripts/make_burntool_pcontrol.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/make_burntool_pcontrol.pl (revision 28003) @@ -1,8 +1,12 @@ -#!/usr/bin/perl -w +#!/usr/bin/env perl # script to generate a pcontrol.pro file to run burntool on a set of data. use DBI; +use warnings; use DateTime; use Getopt::Std; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use IPC::Cmd 0.36 qw( can_run run); use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; @@ -35,7 +39,11 @@ ) or die "Unable to connect to database $DBI::errstr\n"; +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + die "Cannot find required tools."; +} # Determine what the value of "BURNTOOL.STATE.GOOD" currently is: $burntoolStateGood = 999; -open(LAZY,"ppConfigDump -camera GPC1 -dump-camera - |") || die "Can't run ppConfigDump\n"; +open(LAZY,"$ppConfigDump -camera GPC1 -dump-camera - |") || die "Can't run ppConfigDump\n"; while() { chomp; @@ -46,14 +54,54 @@ } close(LAZY); -#unless( $burntoolStateGood == 999) { -# print STDERR "GOOD == $burntoolStateGood\n"; -#} - - -# List of which obs_modes are "science" and which are something else. -%science = ('MD' => 1, '3PI' => 1, 'STS' => 1, 'CAL' => 1, 'M31' => 1, 'SS' => 1, - 'ENGINEERING' => 0, 'NULL' => 0, 'Unknown' => 1, ' ' => 0); + +# Read the nightly science config file and use that to determine which data is science +my $verbose = 0; +my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR); +} + +my @target_list; +my %object_list; +my $comment_list; + +my $mdcParser = PS::IPP::Metadata::Config->new; +my $metadata = $mdcParser->parse(join "", @$stdout_buf); +foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'TARGETS') { + my @target_data = @{ ${ $entry }{value} }; + my $this_target = ''; + + foreach my $tentry (@target_data) { + if (${ $tentry }{name} eq 'NAME') { + $this_target = ${ $tentry }{value}; + push @target_list, $this_target; + $obsmode_list{$this_target} = '.*'; + $object_list{$this_target} = '.*'; + $comment_list{$this_target} = '.*'; + } + elsif (${ $tentry }{name} eq 'OBSMODE') { + $obsmode_list{$this_target} = ${ $tentry }{value}; + $obsmode_list{$this_target} =~ s/%//; + } + elsif (${ $tentry }{name} eq 'OBJECT') { + $object_list{$this_target} = ${ $tentry }{value}; + $object_list{$this_target} =~ s/%//; + } + elsif (${ $tentry }{name} eq 'COMMENT') { + $comment_list{$this_target} = ${ $tentry }{value}; + $comment_list{$this_target} =~ s/%//; + } + } + } +} if (exists($opt{P})) { - $science{Unknown} = 1; + push @target_list, 'PR'; + $obsmode_list{'PR'} = 'Unknown'; + $object_list{'PR'} = '.*'; + $comment_list{'PR'} = '.*'; } @@ -119,5 +167,5 @@ # Find _ALL_ observations within the date range we care about - $sth = "SELECT exp_id,dateobs,pon_time,exp_type,obs_mode,comment FROM rawExp WHERE " . + $sth = "SELECT exp_id,dateobs,pon_time,exp_type,obs_mode,comment,object FROM rawExp WHERE " . "dateobs >= '${date_min}' AND dateobs <= '${date_max}' order by dateobs"; @@ -133,9 +181,12 @@ foreach $row_ref (@{ $data_ref }) { - ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment) = @{ $row_ref }; + ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment,$object) = @{ $row_ref }; # Fix nulls in the database if (!defined($obs_mode)) { $obs_mode = ' '; } + if (!defined($object)) { + $object = ' '; + } if (!defined($comment)) { $comment = ' '; @@ -144,5 +195,5 @@ ($date,$time) = split / /, $dateobs; ($year,$month,$day) = split /-/, $date; - ($hour,$minute,$second) = split /:/, $time; + ($hour,$minute,$second) = split /:/, $time; # / # This stops emacs run-on syntax highlights. $dt = DateTime->new( year => $year, month => $month, day => $day, @@ -155,16 +206,5 @@ $et = $dt->epoch(); - - if (!exists($science{$obs_mode})) { - $science{$obs_mode} = 0; - } - if ((($science{$obs_mode} == 1)&&($comment !~ /Daytime/))|| - (($science{$obs_mode} == 0)&&( - ($comment =~ /MD/)||($comment =~ /ThreePi/)|| - ($comment =~ /STS/i)||($comment =~ /M31/)||($comment =~ /Stellar Transit/)|| - ($comment =~ /CAL/i)||($comment =~ /SS/)||($comment =~ /SVS/)|| - ($comment =~ /Sweetspot/)||($comment =~ /virgo/i)||($comment =~ /kepler/)|| - ($comment =~ /sdss/)))) - { + if (is_science($exp_type,$obs_mode,$object,$comment)) { if ($start_s[-1] == 0) { $start_s[-1] = $et - 1800; @@ -194,5 +234,5 @@ # Scan again and count how many exposures are between windows foreach $row_ref (@{ $data_ref }) { - ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment) = @{ $row_ref }; + ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment,$object) = @{ $row_ref }; if (!defined($obs_mode)) { $obs_mode = ' '; @@ -204,5 +244,5 @@ ($date,$time) = split / /, $dateobs; ($year,$month,$day) = split /-/, $date; - ($hour,$minute,$second) = split /:/, $time; + ($hour,$minute,$second) = split /:/, $time; # / # another emacs syntax highlight bug. $dt = DateTime->new( year => $year, month => $month, day => $day, @@ -215,10 +255,6 @@ $et = $dt->epoch(); - if (!exists($science{$obs_mode})) { - die "No mode >>$obs_mode<<\n"; - } - for ($i = 0; $i <= $#start_s; $i++) { - if (($et < $start_s[$i])&&($science{$obs_mode} == 0)) { + if (($et < $start_s[$i])&&(is_science($exp_type,$obs_mode,$object,$comment) == 0)) { if ($i == 0) { $skips[$i]++; @@ -278,4 +314,30 @@ unless(exists($opt{b}) || ($N_ranges == 0)) { print_epilogue(); +} + + +# Subroutine to use the configuration file data and the exposure data to determine whether or not something is "science" +sub is_science { +# if (is_science($exp_type,$obs_mode,$object,$comment)) { + my ($exp_type,$obs_mode,$object,$comment) = @_; + my $return_value = 0; + unless($object) { + $object = ''; + } + if ($exp_type eq 'OBJECT') { + foreach my $target (@target_list) { +# print STDERR "$target $obsmode_list{$target} $exp_type $obs_mode $object $comment\n"; + if (($obs_mode =~ /$obsmode_list{$target}/)&& + ($object =~ /$object_list{$target}/)&& + ($comment =~ /$comment_list{$target}/)) { + $return_value = 1; + } +# print ">$return_value $exp_type $obs_mode $object $comment $target $obsmode_list{$target} $object_list{$target} $comment_list{$target}\n"; + if ($return_value) { + return($return_value); + } + } + } + return(0); } Index: /branches/pap/ippScripts/scripts/publish_file.pl =================================================================== --- /branches/pap/ippScripts/scripts/publish_file.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/publish_file.pl (revision 28003) @@ -7,5 +7,7 @@ use Sys::Hostname; my $host = hostname(); -print "Starting script $0 on $host\n\n"; +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: /branches/pap/ippScripts/scripts/rcserver_checkstatus.pl =================================================================== --- /branches/pap/ippScripts/scripts/rcserver_checkstatus.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/rcserver_checkstatus.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/receive_advance.pl =================================================================== --- /branches/pap/ippScripts/scripts/receive_advance.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/receive_advance.pl (revision 28003) @@ -7,5 +7,7 @@ use Sys::Hostname; my $host = hostname(); -print "Starting script $0 on $host\n\n"; +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; @@ -58,7 +60,6 @@ &my_die( "Unable to set up", $fileset_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -# update the fileset entry - -if ($dbinfo_uri and ($dbinfo_uri ne "NULL")) { +my $import_run_to_db = 0; +if ($import_run_to_db and $dbinfo_uri and ($dbinfo_uri ne "NULL")) { my $filename = basename($dbinfo_uri); my ($stage) = $filename =~ m|^dbinfo\.(\S+)\.\d+\.mdc$|; # Stage of interest @@ -100,4 +101,5 @@ } +# update the fileset entry # All done { Index: /branches/pap/ippScripts/scripts/receive_file.pl =================================================================== --- /branches/pap/ippScripts/scripts/receive_file.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/receive_file.pl (revision 28003) @@ -7,5 +7,7 @@ use Sys::Hostname; my $host = hostname(); -print "Starting script $0 on $host\n\n"; +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; @@ -124,5 +126,5 @@ print "dirinfo resolved is: $resolved\n" if $verbose; - open OUT, ">$resolved" + open OUT, ">$resolved" or &my_die( "failed to open $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR); print OUT @$dirinfo_lines @@ -130,5 +132,5 @@ close OUT or &my_die( "failed to close $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR); - + # update the fileset to allow processing of other files my $command = "$receivetool -updatefileset -fileset_id $fileset_id"; @@ -160,5 +162,5 @@ &my_die( "failed to resolve $dbinfo_uri\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$resolved; - open OUT, ">$resolved" + open OUT, ">$resolved" or &my_die( "failed to open $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR); @@ -227,5 +229,5 @@ $current_component = $value; $component_dir = $components->{$current_component}; - &my_die( "$component_dir is null for $value in $filename: $runType\n", + &my_die( "$component_dir is null for $value in $filename: $runType\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$component_dir; } elsif ($name eq 'workdir') { @@ -237,5 +239,5 @@ $new_value = basename($value); } elsif ((($name eq 'uri') or ($name eq 'path_base')) and ($value ne 'NULL')) { - &my_die( "$component_dir is null and we need it for $name", + &my_die( "$component_dir is null and we need it for $name", $file_id, $PS_EXIT_PROG_ERROR) if !$component_dir; Index: /branches/pap/ippScripts/scripts/receive_fileset.pl =================================================================== --- /branches/pap/ippScripts/scripts/receive_fileset.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/receive_fileset.pl (revision 28003) @@ -7,5 +7,7 @@ use Sys::Hostname; my $host = hostname(); -print "Starting script $0 on $host\n\n"; +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: /branches/pap/ippScripts/scripts/receive_setstatus.pl =================================================================== --- /branches/pap/ippScripts/scripts/receive_setstatus.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/receive_setstatus.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -74,5 +75,5 @@ $fileargs = " --list - --copy --abspath"; } - + # XXX need to create a fileset type for this my $command = "echo $regline | dsreg --add $status_fs_name --product $status_product --type notset"; Index: /branches/pap/ippScripts/scripts/receive_source.pl =================================================================== --- /branches/pap/ippScripts/scripts/receive_source.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/receive_source.pl (revision 28003) @@ -7,5 +7,7 @@ use Sys::Hostname; my $host = hostname(); -print "Starting script $0 on $host\n\n"; +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: /branches/pap/ippScripts/scripts/register_exp.pl =================================================================== --- /branches/pap/ippScripts/scripts/register_exp.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/register_exp.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -126,4 +127,5 @@ $command .= " -end_stage $end_stage" if defined $end_stage; $command .= " -tess_id $tess_id" if defined $tess_id; +$command .= " -state full"; $command .= " $cmdflags"; Index: /branches/pap/ippScripts/scripts/register_imfile.pl =================================================================== --- /branches/pap/ippScripts/scripts/register_imfile.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/register_imfile.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -115,39 +116,39 @@ my $burntoolStateGood = 0; foreach my $line (split /\n/, $out1) { - if ($line =~ /FPA.BURNTOOL.APPLIED/) { - $line =~ s/^\s+//; - $burntoolState = (split /\s+/, $line)[2]; - } - if ($line =~ /FPA.CAMERA/) { - $line =~ s/^\s+//; - if ((split /\s+/, $line)[2] eq 'GPC1') { - $isGPC1 = 1; - } - } + if ($line =~ /FPA.BURNTOOL.APPLIED/) { + $line =~ s/^\s+//; + $burntoolState = (split /\s+/, $line)[2]; + } + if ($line =~ /FPA.CAMERA/) { + $line =~ s/^\s+//; + if ((split /\s+/, $line)[2] eq 'GPC1') { + $isGPC1 = 1; + } + } } if ($isGPC1 != 1) { - $burntoolState = 0; # If it's not GPC1, you shouldn't have run burntool. + $burntoolState = 0; # If it's not GPC1, you shouldn't have run burntool. } elsif (($isGPC1 == 1) && ($burntoolState == 1)) { -# print STDERR "In the good region: >>$burntoolState<<\n"; - my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - IPC::Cmd::run(command => $ppConfigDump_cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn ("Unable to perform ppConfigDump"); - exit($error_code); - } - - # This is ugly, but doing a full parse for one entry is a bit wasteful. - foreach my $line (split /\n/, (join "", @$stdout_buf)) { - if ($line =~ /BURNTOOL.STATE.GOOD/) { - $line =~ s/^\s+//; - $burntoolStateGood = (split /\s+/, $line)[2]; - last; - } - } - $burntoolState = $burntoolStateGood; # Positive because this has the header table. - +# print STDERR "In the good region: >>$burntoolState<<\n"; + my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + IPC::Cmd::run(command => $ppConfigDump_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform ppConfigDump"); + exit($error_code); + } + + # This is ugly, but doing a full parse for one entry is a bit wasteful. + foreach my $line (split /\n/, (join "", @$stdout_buf)) { + if ($line =~ /BURNTOOL.STATE.GOOD/) { + $line =~ s/^\s+//; + $burntoolStateGood = (split /\s+/, $line)[2]; + last; + } + } + $burntoolState = $burntoolStateGood; # Positive because this has the header table. + } $cmdflags .= " -burntool_state $burntoolState "; @@ -173,4 +174,5 @@ $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; +$command .= " -data_state full"; $command .= " $cmdflags"; Index: /branches/pap/ippScripts/scripts/stack_skycell.pl =================================================================== --- /branches/pap/ippScripts/scripts/stack_skycell.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/stack_skycell.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: /branches/pap/ippScripts/scripts/staticsky.pl =================================================================== --- /branches/pap/ippScripts/scripts/staticsky.pl (revision 28003) +++ /branches/pap/ippScripts/scripts/staticsky.pl (revision 28003) @@ -0,0 +1,324 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); +my $ppSub = can_run('ppSub') or (warn "Can't find ppSub" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($sky_id, $dbname, $threads, $outroot, $reduction, $inverse, $run_state, $verbose, $no_update, $no_op, $redirect, $save_temps); +GetOptions( + 'sky_id=s' => \$sky_id, # Diff identifier + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'run-state=s' => \$run_state, # state for run: 'new' or 'update' + 'outroot=s' => \$outroot, # Output root name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --sky_id --outroot", + -exitval => 3, + ) unless + defined $sky_id and + defined $outroot; + +my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $sky_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +# XXX camera is not known here; cannot use filerules... +# my $logDest = $ipprc->filename("LOG.EXP", $outroot); +my $logDest = "$outroot.log"; +$logDest .= ".update" if $run_state eq "update"; +$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $sky_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF); + +my $outbase = basename($outroot); +my ($listFile, $listName) = tempfile("/tmp/$outbase.list.XXXX", UNLINK => !$save_temps ); + +# Get list of input images to stack photometry +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$staticskytool -inputs -sky_id $sky_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -inputskyfile: $error_code", $sky_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $sky_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $sky_id, $PS_EXIT_PROG_ERROR); +} + +# XXX do we NEED to define the camera? +# &my_die("Unable to identify camera", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless defined $camera; +# $ipprc->define_camera($camera); + +# generate the input +print $listFile "INPUT MULTI\n"; + +foreach my $file (@$files) { + print $listFile "INPUT METADATA\n"; + + $path_base = $file->{path_base}; + my $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $file->{path_base} ); # Mask name + my $maskCnv = $ipprc->filename("PPSTACK.OUTPUT.MASK", $file->{path_base} ); # Mask name + my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name + my $psfCnv = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $file->{path_base} ); # PSF name + + my $imageRaw = $ipprc->filename("PPSTACK.UNCONV", $file->{path_base} ); # Mask name + my $maskRaw = $ipprc->filename("PPSTACK.UNCONV.MASK", $file->{path_base} ); # Mask name + my $weightRaw = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $file->{path_base} ); # Weight name + + my $sources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $file->{path_base}); # Sources name + + &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image ); + &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask ); + &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $weight ); + &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) if ($convolve and not $ipprc->file_exists( $psf )); + &my_die("Sources $sources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $sources ); + + print $listFile " RAW:IMAGE STR " . $imageRaw . "\n"; + print $listFile " RAW:MASK STR " . $maskRaw . "\n"; + print $listFile " RAW:VARIANCE STR " . $weightRaw . "\n"; + + print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; + print $listFile " CNV:MASK STR " . $maskCnv . "\n"; + print $listFile " CNV:VARIANCE STR " . $weightCnv . "\n"; + print $listFile " CNV:PSF STR " . $psfCnv . "\n"; + + print $listFile " SOURCES STR " . $sources . "\n"; + + print $listFile "END\n\n"; + + &my_die("Couldn't find input: $imageRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageRaw"); + &my_die("Couldn't find input: $maskRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskRaw"); + &my_die("Couldn't find input: $weightRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightRaw"); + &my_die("Couldn't find input: $imageCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv"); + &my_die("Couldn't find input: $maskCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv"); + &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv"); + &my_die("Couldn't find input: $psfCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); + &my_die("Couldn't find input: $sources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources"); +} + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT'); # Recipe to use for psphot +unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for STACKPHOT: $reduction\n", $sky_id, $PS_EXIT_CONFIG_ERROR); +} + +print "reduction: $reduction\n"; +print "recipe_psphot: $recipe_psphot\n"; + +# Get the output filenames +my $outputName = $ipprc->filename("PPSUB.OUTPUT", $outroot); +my $outputMask = $ipprc->filename("PPSUB.OUTPUT.MASK", $outroot); +my $outputVariance = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $outroot); +my $outputSources = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $outroot); +my $jpeg1Name = $ipprc->filename("PPSUB.OUTPUT.JPEG1", $outroot); +my $jpeg2Name = $ipprc->filename("PPSUB.OUTPUT.JPEG2", $outroot); +my $configuration = $ipprc->filename("PPSUB.CONFIG", $outroot); +my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); + +if ($run_state eq 'update') { + $traceDest .= '.update'; + $outputStats .= '.update'; +} + +my ($inverseName, $inverseMask, $inverseVariance, $inverseSources); +if ($inverse) { + $inverseName = $ipprc->filename("PPSUB.INVERSE", $outroot); + $inverseMask = $ipprc->filename("PPSUB.INVERSE.MASK", $outroot); + $inverseVariance = $ipprc->filename("PPSUB.INVERSE.VARIANCE", $outroot); + $inverseSources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $outroot); +} + +my $cmdflags; + +# Perform subtraction +{ + my $command = "$ppSub $outroot"; + $command .= " -inimage $input"; + $command .= " -refimage $template"; + $command .= " -inmask $inputMask"; + $command .= " -refmask $templateMask"; + $command .= " -invariance $inputVariance"; + $command .= " -refvariance $templateVariance"; + $command .= " -insources $inputSources"; + $command .= " -refsources $templateSources"; + $command .= " -stats $outputStats"; + $command .= " -threads $threads" if defined $threads; + if ($run_state eq "new") { + $command .= " -dumpconfig $configuration"; + } else { + my $configurationReal = $ipprc->file_resolve($configuration) or &my_die("Couldn't resolve configuration file: $configuration", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR); + $command .= " -ipprc $configurationReal"; + } + $command .= " -save-inconv" if defined $saveInConv; + $command .= " -save-refconv" if defined $saveRefConv; + $command .= " -recipe PPSUB $recipe_ppSub"; + $command .= " -recipe PSPHOT $recipe_psphot"; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; + $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF"; + if ($run_state eq "new") { + $command .= " -photometry"; + } + $command .= " -inverse" if $inverse; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id; + $command .= " -source_id $source_id" if defined $source_id; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppSub: $error_code", $sky_id, $skycell_id, $error_code); + } + + my $outputStatsReal = $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputStats", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); + + # measure chip stats + $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $sky_id, $skycell_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + + my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + + if (!$quality) { + &my_die("Couldn't find expected output file: $outputName", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + &my_die("Couldn't find expected output file: $outputMask", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputVariance", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance); + &my_die("Couldn't find expected output file: $outputSources", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + &my_die("Couldn't find expected output file: $jpeg1Name", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg1Name); + &my_die("Couldn't find expected output file: $jpeg2Name", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg2Name); + if ($inverse) { + &my_die("Couldn't find expected output file: $inverseName", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseName); + &my_die("Couldn't find expected output file: $inverseMask", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseMask); + &my_die("Couldn't find expected output file: $inverseVariance", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseVariance); + &my_die("Couldn't find expected output file: $inverseSources", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseSources); + if ($run_state eq 'new') { + &my_die("Couldn't find expected output file: $configuration", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration); + } + } + } + } else { + print "Not executing: $command\n"; + } +} + +unless ($no_update) { + + # Add the subtraction result + { + my $command = "$difftool -sky_id $sky_id -skycell_id $skycell_id"; + $command .= " -magicked $magicked" if $magicked; + if ($run_state eq 'new') { + $command .= " -adddiffskyfile -path_base $outroot"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + } else { + $command .= " -tofullskyfile"; + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = $run_state eq "update" ? + "Unable to perform difftool -adddiffskyfile" : + "Unable to perform difftool -tofullskyfile"; + &my_die("$err_message: $error_code", $sky_id, $skycell_id, $error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $sky_id = shift; # Diff identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $sky_id and defined $skycell_id and not $no_update) { + my $command = "$difftool -sky_id $sky_id -skycell_id $skycell_id -fault $exit_code"; + if ($run_state eq 'new') { + $command .= " -adddiffskyfile"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + } else { + $command .= " -updatediffskyfile"; + } + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/pap/ippScripts/scripts/tdl_generate.pl =================================================================== --- /branches/pap/ippScripts/scripts/tdl_generate.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/tdl_generate.pl (revision 28003) @@ -13,6 +13,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; #print "\n\n"; -#print "Starting script $0 on $host\n\n"; +#print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -58,5 +59,5 @@ # resolve any path:// or file:// in outroot $outroot = $ipprc->file_resolve($outroot); - + # Look for programs we need my $missing_tools; Index: /branches/pap/ippScripts/scripts/warp_overlap.pl =================================================================== --- /branches/pap/ippScripts/scripts/warp_overlap.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/warp_overlap.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: /branches/pap/ippScripts/scripts/warp_skycell.pl =================================================================== --- /branches/pap/ippScripts/scripts/warp_skycell.pl (revision 28002) +++ /branches/pap/ippScripts/scripts/warp_skycell.pl (revision 28003) @@ -7,6 +7,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; @@ -299,5 +300,5 @@ $command .= " -tess_id $tess_dir"; $command .= " -path_base $outroot"; # needed for logfile lookups - $command .= " -magicked $magicked" if $magicked; + $command .= " -set_magicked $magicked" if $magicked; $command .= " -uri $outputImage" if !$quality; @@ -321,4 +322,5 @@ $command .= " -warp_id $warp_id"; $command .= " -skycell_id $skycell_id"; + $command .= " -set_magicked $magicked" if $magicked; $command .= " -dbname $dbname" if defined $dbname; Index: /branches/pap/ippTasks/camera.pro =================================================================== --- /branches/pap/ippTasks/camera.pro (revision 28002) +++ /branches/pap/ippTasks/camera.pro (revision 28003) @@ -154,4 +154,5 @@ book getword camPendingExp $pageName cam_id -var CAM_ID book getword camPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword camPendingExp $pageName path_base -var PATH_BASE book getword camPendingExp $pageName dvodb -var DVODB book getword camPendingExp $pageName dbname -var DBNAME @@ -173,6 +174,10 @@ # neb:///ipp004-vol0/gpc1/20080130 - ## generate outroot specific to this exposure (& chip) - sprintf outroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID + if ("$PATH_BASE" == "NULL") + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID + else + $outroot = $PATH_BASE + end stdout $LOGDIR/camera.exp.log Index: /branches/pap/ippTasks/chip.pro =================================================================== --- /branches/pap/ippTasks/chip.pro (revision 28002) +++ /branches/pap/ippTasks/chip.pro (revision 28003) @@ -153,4 +153,5 @@ book getword chipPendingImfile $pageName reduction -var REDUCTION book getword chipPendingImfile $pageName state -var RUN_STATE + book getword chipPendingImfile $pageName path_base -var PATH_BASE if ($RAW_MAGICKED > 0) @@ -174,6 +175,10 @@ # neb:///ipp004-vol0/gpc1/20080130 - ## generate outroot specific to this exposure (& chip) - sprintf outroot "%s/%s/%s.ch.%s" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID + if ("$PATH_BASE" == "NULL") + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.ch.%s" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID + else + $outroot = $PATH_BASE + end stdout $LOGDIR/chip.imfile.log Index: /branches/pap/ippTasks/destreak.pro =================================================================== --- /branches/pap/ippTasks/destreak.pro (revision 28002) +++ /branches/pap/ippTasks/destreak.pro (revision 28003) @@ -37,4 +37,6 @@ echo magicDSToRevert book listbook magicDSToRevert + echo magicDSToCleanup + book listbook magicDSToCleanup end @@ -212,4 +214,5 @@ book getword magicToDS $pageName recoveryroot -var RECROOT book getword magicToDS $pageName re_place -var REPLACE + book getword magicToDS $pageName magicked -var MAGICKED book getword magicToDS $pageName dbname -var DBNAME @@ -226,5 +229,5 @@ # TODO: do not add recoveryroot or replace if they are null or zero - $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE + $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED add_standard_args run @@ -273,5 +276,5 @@ #periods -exec $LOADEXEC periods -exec 30 - periods -timeout 20 + periods -timeout 60 npending 1 @@ -412,4 +415,5 @@ book getword magicDSToRevert $pageName re_place -var REPLACE book getword magicDSToRevert $pageName bothways -var BOTHWAYS + book getword magicDSToRevert $pageName magicked -var MAGICKED book getword magicDSToRevert $pageName dbname -var DBNAME @@ -424,5 +428,5 @@ end - $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --replace $REPLACE --bothways $BOTHWAYS + $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --replace $REPLACE --bothways $BOTHWAYS --magicked $MAGICKED add_standard_args run Index: /branches/pap/ippTasks/diff.pro =================================================================== --- /branches/pap/ippTasks/diff.pro (revision 28002) +++ /branches/pap/ippTasks/diff.pro (revision 28003) @@ -184,4 +184,5 @@ book getword diffSkyfile $pageName reduction -var REDUCTION book getword diffSkyfile $pageName diff_mode -var DIFF_MODE + book getword diffSkyfile $pageName path_base -var PATH_BASE # set the host and workdir based on the skycell hash @@ -207,5 +208,10 @@ basename $TESS_DIR -var TESS_ID - sprintf outroot "%s/%s/%s/%s.%s.%sdif.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $DIFF_TAG $DIFF_ID + + if ("$PATH_BASE" == "NULL") + sprintf outroot "%s/%s/%s/%s.%s.%sdif.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $DIFF_TAG $DIFF_ID + else + $outroot = $PATH_BASE + end stdout $LOGDIR/diff.skycell.log Index: /branches/pap/ippTasks/nightly_stacks.pro =================================================================== --- /branches/pap/ippTasks/nightly_stacks.pro (revision 28002) +++ /branches/pap/ippTasks/nightly_stacks.pro (revision 28003) @@ -164,10 +164,12 @@ task.exec + stdout $LOGDIR/ns.initday.log + stderr $LOGDIR/ns.initday.log $today = `date +%Y-%m-%d` book newpage nsData $today book setword nsData $today nsState NEW - command echo $today -# command automate_stacks.pl --clean_old --date $today +# command echo $today + command automate_stacks.pl --clean_old --date $today end @@ -370,5 +372,5 @@ periods -poll $LOADPOLL periods -exec $LOADEXEC - periods -timeout 30 + periods -timeout 120 npending 1 @@ -424,5 +426,5 @@ periods -poll $LOADPOLL periods -exec $LOADEXEC - periods -timeout 30 + periods -timeout 120 npending 1 Index: /branches/pap/ippTasks/pstamp.pro =================================================================== --- /branches/pap/ippTasks/pstamp.pro (revision 28002) +++ /branches/pap/ippTasks/pstamp.pro (revision 28003) @@ -13,4 +13,6 @@ $pstampFin_DB = 0 $pstampRev_DB = 0 +$pstampDep_DB = 0 +$pstampCleanup_DB = 0 # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config @@ -28,8 +30,9 @@ book init pstampJob book init pstampFinish + book init pstampDependent + book init pstampCleanup end pstamp.reset - macro pstamp.on @@ -57,5 +60,5 @@ active true end - task pstamp.job.revert + task pstamp.dependent.load active true end @@ -89,4 +92,10 @@ active false end + task pstamp.dependent.load + active false + end + task pstamp.dependent.run + active false + end end @@ -98,4 +107,20 @@ macro pstamp.revert.off task pstamp.job.revert + active false + end +end +macro pstamp.cleanup.on + task pstamp.cleanup.load + active true + end + task pstamp.cleanup.run + active true + end +end +macro pstamp.cleanup.off + task pstamp.cleanup.run + active false + end + task pstamp.cleanup.run active false end @@ -150,5 +175,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.request.load.log + stdout NULL stderr $LOGSUBDIR/pstamp.request.load.log $run = pstamptool -pendingreq @@ -195,5 +220,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.request.run.log + stdout NULL stderr $LOGSUBDIR/pstamp.request.run.log book npages pstampRequest -var N @@ -208,8 +233,10 @@ book getword pstampRequest $pageName uri -var URI book getword pstampRequest $pageName ds_outProduct -var PRODUCT + book getword pstampRequest $pageName outdir -var OUTDIR + book getword pstampRequest $pageName label -var LABEL host anyhost - $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --redirect-output + $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --outdir $OUTDIR --label $LABEL --redirect-output add_standard_args run @@ -252,5 +279,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.finish.load.log + stdout NULL stderr $LOGSUBDIR/pstamp.finish.load.log $run = pstamptool -completedreq @@ -297,5 +324,5 @@ task.exec - stdout $LOGSUBDIR/request.finish.run.log + stdout NULL stderr $LOGSUBDIR/request.finish.run.log book npages pstampFinish -var N @@ -312,6 +339,7 @@ book getword pstampFinish $pageName name -var REQ_NAME book getword pstampFinish $pageName outProduct -var PRODUCT - - $run = request_finish.pl --req_id $REQ_ID --req_type $REQ_TYPE --req_file $URI --req_name $REQ_NAME --product $PRODUCT --redirect-output + book getword pstampFinish $pageName outdir -var OUTDIR + + $run = request_finish.pl --req_id $REQ_ID --req_type $REQ_TYPE --req_file $URI --req_name $REQ_NAME --product $PRODUCT --outdir $OUTDIR --redirect-output add_standard_args run @@ -354,5 +382,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.job.load.log + stdout NULL stderr $LOGSUBDIR/pstamp.job.load.log $run = pstamptool -pendingjob @@ -402,5 +430,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.job.run.log + stdout NULL stderr $LOGSUBDIR/pstamp.job.run.log book npages pstampJob -var N @@ -480,5 +508,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.job.revert.log + stdout NULL stderr $LOGSUBDIR/pstamp.job.revert.log $run = pstamptool -revertjob -all @@ -512,2 +540,257 @@ end +task pstamp.dependent.load + host local + + periods -poll $LOADPOLL + # XXX: create a macro for this time + periods -exec 10 +# periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.dependent.load.log + $run = pstamptool -pendingdependent + if ($DB:n == 0) + option DEFAULT + else + option $DB:$pstampDep_DB + $run = $run -dbname $DB:$pstampDep_DB $PS_DBSERVER + $pstampDep_DB ++ + if ($pstampDep_DB >= $DB:n) set pstampDep_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampDependent -key dep_id -uniq -setword dbname $options:0 -setword pantaskState INIT + + book npages pstampDependent -var N + if ($VERBOSE > 2) + book listbook pstampDependent + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pstampDependent + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end + + task.exit default + showcommand failure + end + +end + +task pstamp.dependent.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + + task.exec + book npages pstampDependent -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + periods -exec 0.05 + + book getpage pstampDependent 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pstampDependent $pageName pantaskState RUN + book getword pstampDependent $pageName dep_id -var DEP_ID + book getword pstampDependent $pageName state -var STATE + book getword pstampDependent $pageName stage -var STAGE + book getword pstampDependent $pageName stage_id -var STAGE_ID + book getword pstampDependent $pageName component -var COMPONENT + book getword pstampDependent $pageName imagedb -var IMAGEDB + book getword pstampDependent $pageName rlabel -var RLABEL + book getword pstampDependent $pageName outdir -var OUTDIR + book getword pstampDependent $pageName need_magic -var NEED_MAGIC + book getword pstampDependent $pageName dbname -var DBNAME + + if ($VERBOSE > 1) + book listpage pstampDependent $pageName + end + + host anyhost + + if ("$NEED_MAGIC" == "T") + $NEED_MAGIC="--need_magic" + else + $NEED_MAGIC="" + end + + $MYLOGFILE = $OUTDIR/checkdep.$DEP_ID.log + stdout $MYLOGFILE + stderr $MYLOGFILE + + $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC + + add_standard_args run + + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + + task.exit $EXIT_SUCCESS + if ($VERBOSE > 1) + echo pstamp.dependent.run task.exit $DEP_ID status: $JOB_STATUS + end + process_exit pstampDependent $options:0 $JOB_STATUS + end + task.exit default + if ($VERBOSE > 1) + echo pstamp.job.run task.exit $DEP_ID status: $JOB_STATUS + end + showcommand failure + process_exit pstampDependent $options:0 $JOB_STATUS + end + + task.exit crash + echo pstamp.job.run task.crash $DEP_ID status: $JOB_STATUS + process_exit pstampDependent $options:0 $JOB_STATUS + showcommand crash + book setword pstampDependent $options:0 pantaskState CRASH + end + + task.exit timeout + echo pstamp.job.run task.timeout $DEP_ID status: $JOB_STATUS + process_exit pstampDependent $options:0 $JOB_STATUS + showcommand timeout + book setword pstampDependent $options:0 pantaskState TIMEOUT + end +end + +task pstamp.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.cleanup.load.log + $run = pstamptool -pendingcleanup + if ($DB:n == 0) + option DEFAULT + else + option $DB:$pstampCleanup_DB + $run = $run $PS_DBSERVER -dbname $DB:$pstampCleanup_DB + $pstampCleanup_DB ++ + if ($pstampCleanup_DB >= $DB:n) set pstampCleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampCleanup -key req_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + echo starting request + book listbook pstampCleanup + end + + process_cleanup pstampCleanup + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + # since everything is on one file system keep npending low to avoid + # overloading nfs + + npending 10 + + task.exec + book npages pstampCleanup -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + periods -exec 0.05 + + book getpage pstampCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pstampCleanup $pageName pantaskState RUN + book getword pstampCleanup $pageName req_id -var REQ_ID + book getword pstampCleanup $pageName dbname -var DBNAME + book getword pstampCleanup $pageName name -var NAME + book getword pstampCleanup $pageName outdir -var OUTDIR + book getword pstampCleanup $pageName uri -var URI + book getword pstampCleanup $pageName reqType -var REQTYPE + book getword pstampCleanup $pageName outProduct -var PRODUCT + + # XXX: have the script set this up this + $MYLOGFILE=/data/ippdb02.0/pstamp/work/logs/cleanup.$REQ_ID + stdout $MYLOGFILE + stderr $MYLOGFILE + + host anyhost + + $run = pstamp_cleanup.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --name $NAME --outdir $OUTDIR --reqType $REQTYPE + + add_standard_args run + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + + task.exit $EXIT_SUCCESS + process_exit pstampCleanup $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pstampCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword pstampCleanup $options:0 pantaskState CRASH + end + + task.exit timeout + showcommand timeout + book setword pstampCleanup $options:0 pantaskState TIMEOUT + end +end + Index: /branches/pap/ippTasks/simtest.simmosaic.config =================================================================== --- /branches/pap/ippTasks/simtest.simmosaic.config (revision 28003) +++ /branches/pap/ippTasks/simtest.simmosaic.config (revision 28003) @@ -0,0 +1,114 @@ + +CAMERA STR SIMMOSAIC + +FILERULE MULTI +FILERULE STR Chip00.fits +FILERULE STR Chip01.fits +FILERULE STR Chip10.fits +FILERULE STR Chip11.fits + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +SEQUENCE METADATA + OBSTYPE STR OBJECT + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + + +SEQUENCE METADATA + OBSTYPE STR OBJECT + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 1 + DITHER.ND S32 1 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,r,i,i + @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 + @SKYMAGS F32 20.86,20.86,20.15,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/pap/ippTasks/stack.pro =================================================================== --- /branches/pap/ippTasks/stack.pro (revision 28002) +++ /branches/pap/ippTasks/stack.pro (revision 28003) @@ -116,5 +116,5 @@ end add_poll_args run - # add_poll_labels run + add_poll_labels run command $run end @@ -172,4 +172,5 @@ book getword stackSumSkyfile $pageName skycell_id -var SKYCELL_ID book getword stackSumSkyfile $pageName workdir -var WORKDIR_TEMPLATE + book getword stackSumSkyfile $pageName path_base -var PATH_BASE book getword stackSumSkyfile $pageName reduction -var REDUCTION book getword stackSumSkyfile $pageName dbname -var DBNAME @@ -185,5 +186,9 @@ basename $TESS_DIR -var TESS_ID - sprintf outroot "%s/%s/%s/%s.%s.stk.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $STACK_ID + if ("$PATH_BASE" == "NULL") + sprintf outroot "%s/%s/%s/%s.%s.stk.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $STACK_ID + else + $outroot = $PATH_BASE + end stdout $LOGDIR/stack.skycell.log Index: /branches/pap/ippTasks/summit.copy.pro =================================================================== --- /branches/pap/ippTasks/summit.copy.pro (revision 28002) +++ /branches/pap/ippTasks/summit.copy.pro (revision 28003) @@ -107,6 +107,9 @@ periods -timeout 20 npending 1 - # trange 16:00 23:59 - # trange 00:00 04:00 + + # only active in the day (06:00 to 19:00 HST, times are UT): + trange -reset + # trange 16:00 23:59 + # trange 00:00 05:00 task.exec @@ -153,11 +156,15 @@ # and inserting these into a db table on the local cluster (pzDownloadExp) task pzgetexp + host local + periods -exec 30 periods -poll 1 periods -timeout 700 - # trage 16:00 23:59 - # trage 00:00 04:00 npending 1 - host local + + # only active in the day (06:00 to 19:00 HST, times are UT): + trange -reset + # trange 16:00 23:59 + # trange 00:00 05:00 task.exec @@ -213,7 +220,10 @@ periods -poll 1 periods -timeout 20 - # trange 16:00 23:59 - # trange 00:00 04:00 npending 1 + + # only active in the day (06:00 to 19:00 HST, times are UT): + trange -reset + # trange 16:00 23:59 + # trange 00:00 05:00 task.exec @@ -224,5 +234,5 @@ # save the DB name for the exit tasks option $DB:$pztoolPendingExp_DB - command pztool -pendingexp -limit 5 -dbname $DB:$pztoolPendingExp_DB + command pztool -pendingexp -limit 10 -dbname $DB:$pztoolPendingExp_DB $pztoolPendingExp_DB ++ if ($pztoolPendingExp_DB >= $DB:n) set pztoolPendingExp_DB = 0 @@ -261,11 +271,15 @@ # database table of imfiles task pzgetimfile + host local + periods -exec 0.05 periods -poll 0.025 periods -timeout 700 - # trage 16:00 23:59 - # trage 00:00 04:00 - host local npending 10 + + # only active in the day (06:00 to 19:00 HST, times are UT): + trange -reset + # trange 16:00 23:59 + # trange 00:00 05:00 task.exec @@ -336,10 +350,13 @@ host local - periods -exec 30 + periods -exec 10 periods -poll 1 periods -timeout 120 - # trage 16:00 23:59 - # trage 00:00 04:00 npending 1 + + # only active in the day (06:00 to 19:00 HST, times are UT): + trange -reset + # trange 16:00 23:59 + # trange 00:00 05:00 # select entries from the current DB; cycle to the next DB, if it exists @@ -352,5 +369,5 @@ # save the DB name for the exit tasks option $DB:$pztoolPendingImfile_DB - command pztool -pendingimfile -limit 60 -dbname $DB:$pztoolPendingImfile_DB + command pztool -pendingimfile -limit 240 -dbname $DB:$pztoolPendingImfile_DB $pztoolPendingImfile_DB ++ if ($pztoolPendingImfile_DB >= $DB:n) set pztoolPendingImfile_DB = 0 @@ -384,10 +401,9 @@ periods -poll 0.05 periods -timeout 1150 + + # only active in the day (06:00 to 19:00 HST, times are UT): trange -reset - # only active in the night (18:00 to 06:00 HST, times are UT): - # trange 04:00 16:00 - # only active in the day (06:00 to 18:00 HST, times are UT): # trange 16:00 23:59 - # trange 00:00 04:00 + # trange 00:00 05:00 task.exec @@ -548,6 +564,4 @@ periods -poll 1 periods -timeout 120 - # trage 16:00 23:59 - # trage 00:00 04:00 npending 1 @@ -592,6 +606,4 @@ periods -poll 0.05 periods -timeout 650 - # trage 16:00 23:59 - # trage 00:00 04:00 task.exec Index: /branches/pap/ippTasks/warp.pro =================================================================== --- /branches/pap/ippTasks/warp.pro (revision 28002) +++ /branches/pap/ippTasks/warp.pro (revision 28003) @@ -314,4 +314,5 @@ book getword warpPendingSkyCell $pageName camera -var CAMERA book getword warpPendingSkyCell $pageName workdir -var WORKDIR_TEMPLATE + book getword warpPendingSkyCell $pageName path-base -var PATH_BASE book getword warpPendingSkyCell $pageName dbname -var DBNAME # XXX change tess_id to tess_dir when schema is changed @@ -331,6 +332,10 @@ set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR - ## generate outroot specific to this exposure - sprintf outroot "%s/%s/%s.wrp.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID $SKYCELL_ID + if ("$PATH_BASE" == "NULL") + ## generate outroot specific to this exposure + sprintf outroot "%s/%s/%s.wrp.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID $SKYCELL_ID + else + $outroot = $PATH_BASE + end stdout $LOGDIR/warp.skycell.log Index: /branches/pap/ippToPsps/config/currentJobId.txt =================================================================== --- /branches/pap/ippToPsps/config/currentJobId.txt (revision 28002) +++ /branches/pap/ippToPsps/config/currentJobId.txt (revision 28003) @@ -1,1 +1,1 @@ -733 +5580 Index: /branches/pap/ippToPsps/config/detection/map.xml =================================================================== --- /branches/pap/ippToPsps/config/detection/map.xml (revision 28002) +++ /branches/pap/ippToPsps/config/detection/map.xml (revision 28003) @@ -5,34 +5,36 @@ - + - - + + - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + +
@@ -48,7 +50,7 @@ + - @@ -59,27 +61,29 @@ - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + Index: /branches/pap/ippToPsps/config/detection/tables.xml =================================================================== --- /branches/pap/ippToPsps/config/detection/tables.xml (revision 28002) +++ /branches/pap/ippToPsps/config/detection/tables.xml (revision 28003) @@ -17,34 +17,36 @@ - + - - + + - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + @@ -93,27 +95,29 @@ - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + @@ -127,5 +131,5 @@ - + Index: /branches/pap/ippToPsps/config/init/data.xml =================================================================== --- /branches/pap/ippToPsps/config/init/data.xml (revision 28002) +++ /branches/pap/ippToPsps/config/init/data.xml (revision 28003) @@ -31,5 +31,5 @@ - + @@ -64,13 +64,4 @@ -
- - - - - - - -
@@ -101472,6 +101463,314 @@ - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + Index: /branches/pap/ippToPsps/config/init/tables.xml =================================================================== --- /branches/pap/ippToPsps/config/init/tables.xml (revision 28002) +++ /branches/pap/ippToPsps/config/init/tables.xml (revision 28003) @@ -1,9 +1,9 @@ - + - +
@@ -61,6 +61,6 @@ - - + +
@@ -90,12 +90,12 @@ - - - - - - - - + + + + + + + +
@@ -104,5 +104,5 @@ - +
Index: /branches/pap/ippToPsps/config/test/map.xml =================================================================== --- /branches/pap/ippToPsps/config/test/map.xml (revision 28003) +++ /branches/pap/ippToPsps/config/test/map.xml (revision 28003) @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + +
+ + Index: /branches/pap/ippToPsps/config/test/tables.xml =================================================================== --- /branches/pap/ippToPsps/config/test/tables.xml (revision 28003) +++ /branches/pap/ippToPsps/config/test/tables.xml (revision 28003) @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ Index: /branches/pap/ippToPsps/scripts/convertPhotCodesToXml.pl =================================================================== --- /branches/pap/ippToPsps/scripts/convertPhotCodesToXml.pl (revision 28003) +++ /branches/pap/ippToPsps/scripts/convertPhotCodesToXml.pl (revision 28003) @@ -0,0 +1,84 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; +use IPC::Cmd 0.36 qw( can_run run ); +use XML::Writer; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $inputPath = undef; +my $outputPath = "photcodes.xml"; + +# get user args +GetOptions( + 'input|i=s' => \$inputPath, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +# tell off user for not providing proper args +pod2usage( + -msg => "\n Required options:\n\n". + "--input \n". + -exitval => 3 + ) unless +defined $inputPath; + + + +my $output = new IO::File(">$outputPath"); +my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2); +$writer->xmlDecl('UTF-8'); + +$writer->startTag('table', "name" => "PhotoCal"); + +open (PHOTCODES, $inputPath); +my $photCode; +my $zeroPoint; +my $filter; +my $description; + +while () { + + chomp; + + if ($_ =~ m/\s+([0-9]+)\s+GPC1.*/) { + + my @columns = split(/\s+/, $_); + + my $filter = $columns[11]; + my $description = $columns[2]; + my $photCode = $columns[1]; + my $zeroPoint = $columns[4]; + + $writer->startTag('row', + "photoCalID" => $photCode, + "filterID" => $filter, + "photoCodeDesc" => $description, + "AB" => "0.0", # TODO + "zeropoint" => $zeroPoint, + "extinction" => "0.0", # TODO + "colorterm" => "0.0", # TODO + "colorExtn" => "0.0", # TODO + "orphanCalColor" => "0.0", # TODO + "orphanCalColorErr" => "0.0", # TODO + "startDate" => "54000."); + + $writer->endTag(); + + + + } +} + +$writer->endTag(); + + +close PHOTCODES; + +# finish up XML +#$writer->endTag(); +$writer->end(); + + Index: /branches/pap/ippToPsps/scripts/getExpIdFromJob.pl =================================================================== --- /branches/pap/ippToPsps/scripts/getExpIdFromJob.pl (revision 28003) +++ /branches/pap/ippToPsps/scripts/getExpIdFromJob.pl (revision 28003) @@ -0,0 +1,65 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); + +my $path = undef; +my $list = undef; +my $job = undef; + +GetOptions( + 'path|p=s' => \$path, + 'list|l=s' => \$list, + 'job|j=s' => \$job, + ) or pod2usage( 2 ); + + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "\n Required options:\n\n". + "--path \n". + "--list | --job \n\n", + -exitval => 3 + ) unless +defined $path and +(defined $list || defined $job); + +if (defined $job) {getExpId($job);} +else { + + open (LIST, "$list") || die "couldn't open the file at '$list'"; + + while ($job = ) { + + getExpId($job); + + } + +close(LIST); +} +################################################################# +# +# +################################################################# +sub getExpId { + my ($job) = @_; + + chomp($job); + + my @cmdOut = `tar -ztf $path/$job/B000.tar.gz`; + + my $line; + foreach $line (@cmdOut) { + + if ($line =~ m/.*B000\/(\d+)\.FITS/) { + + print "$1\n"; + + } + + } +} + Index: /branches/pap/ippToPsps/scripts/ippToPsps_run.pl =================================================================== --- /branches/pap/ippToPsps/scripts/ippToPsps_run.pl (revision 28002) +++ /branches/pap/ippToPsps/scripts/ippToPsps_run.pl (revision 28003) @@ -12,5 +12,5 @@ # globals -my ($verbose, $save_temps, $no_op, $no_update, $camera, $product, $output, $dvodb, $label, $singleExpId, $no_publish); +my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $label, $singleExpId, $no_publish); # TODO temporary until we use database to store current jobid @@ -26,12 +26,14 @@ $output = undef; $singleExpId = undef; +$datastoreProduct = "PSPS_test"; # get user args GetOptions( 'output|o=s' => \$output, - 'product|p=s' => \$product, + 'batch|b=s' => \$batchType, 'dvodb|d=s' => \$dvodb, 'label|l=s' => \$label, 'expid|e=s' => \$singleExpId, + 'product|p=s' => \$datastoreProduct, 'no_publish' => \$no_publish, 'verbose|v' => \$verbose, @@ -46,11 +48,17 @@ pod2usage( -msg => "\n Required options:\n\n". - "--product \n". + "--batch \n". "--output \n". - "--label