Index: /trunk/tools/eam/rawfix.20230221/src/ipp_apply_burntool_psvideo.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/ipp_apply_burntool_psvideo.pl (revision 42458) +++ /trunk/tools/eam/rawfix.20230221/src/ipp_apply_burntool_psvideo.pl (revision 42458) @@ -0,0 +1,278 @@ +#!/usr/bin/env perl + +# this program is used to run 'burntool' on images for a single chip +# for all exposures for a given time period. +# USAGE: ipp_apply_burntool.pl --dbname gpc1 --class_id XY00 --dateobs_begin YYYY/MM/DD --dateobs_end YYYY/MM/DD + +use warnings; +use strict; +use Carp; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $class_id, $dateobs_begin, $dateobs_end, $convert, $dbname, $logfile, $verbose, $save_temps, $rerun, $psvidOutput); +GetOptions( + 'class_id=s' => \$class_id, # chip identifier + 'dateobs_begin=s' => \$dateobs_begin, # exposure date/time range start + 'dateobs_end=s' => \$dateobs_end, # exposure date/time range stop + 'dbname|d=s' => \$dbname, # Database name + 'psvidOutput|d=s' => \$psvidOutput, # output base for psvideo data + 'logfile=s' => \$logfile, + 'rerun' => \$rerun, + 'verbose' => \$verbose, # Print to stdout + 'save-temps' => \$save_temps, # Save temporary files? + + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --class_id --dateobs_begin --dateobs_end --dbname", + -exitval => 3, + ) unless + defined $class_id and + defined $dateobs_begin and + defined $dateobs_end and + defined $dbname; + +# burntool is only valid for GPC1, do not allow substitutes +my $camera = "GPC1"; + +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $funpack = can_run('funpack') or (warn "Can't find funpack" and $missing_tools = 1); +my $burntool = can_run('burntool') or (warn "Can't find burntool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $nebXattr = can_run('neb-xattr') or (warn "Can't find neb-xattr" and $missing_tools = 1); +my $nebreplicate = can_run('neb-replicate') or (warn "Can't find neb-replicate" and $missing_tools = 1); +my $psvideostats = can_run('psvideostats.sh') or (warn "Can't find psvideostats.sh" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# XXX check we can access / create directory named $psvidOutput + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# IPP configuration (including nebulous) +my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up"); + +$ipprc->redirect_output($logfile) if $logfile; + +# Determine the value of a "good" burntool run. +# XXX this program should always generate the latest version +# but the processing system must be told to use an earlier version +# until the newer version is validated. Perhaps split these. +my $config_cmd = "$ppConfigDump -camera $camera -get-key BURNTOOL.STATE.GOOD"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $config_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR); +} + +my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR); + +my $burntoolStateGood = 999; +foreach my $cfg (@$recipeData) { + if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') { + $burntoolStateGood = $cfg->{value}; + } +} +if ($burntoolStateGood == 999) { + &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR); +} +my $outState = -1 * abs($burntoolStateGood); + +print "Target burntool version: $burntoolStateGood\n"; + +# Define list of images to examine. +my $command = "$regtool -processedimfile"; +$command .= " -class_id $class_id"; +$command .= " -dateobs_begin $dateobs_begin"; +$command .= " -dateobs_end $dateobs_end"; +$command .= " -ordered_by_date"; +$command .= " -dbname $dbname" if defined $dbname; + +( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); +} + +# EAM : I am not sure why this is implemented in an esoteric way rather than using the MD parsing API +my @whole = split /\n/, (join "", @$stdout_buf); +my @single = (); +my $files; +while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ); + &my_die("Unable to parse output from regtool", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_PROG_ERROR) unless + defined $list; + push @{ $files }, @$list; + @single = (); + } +} + +#my $metadata = $mdcParser->parse(join "", @$stdout_buf) or +# &my_die("Unable to parse metadata", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); +# +#my $files = parse_md_list($metadata) or +# &my_die("Unable to parse metadata list", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); + +my $REALRUN = 1; + +# Process each file in turn, checking to see if it needs processing, and doing it all in a single pass +my $processNext = 0; +my $previousTable = ''; + +foreach my $file (@$files) { + my $exp_id = $file->{exp_id}; + my $state = $file->{burntool_state}; + + my $rawImfile = $file->{uri}; + my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0); + + my $outTable = $file->{uri}; + + if($burntoolStateGood == 15) { + $outTable =~ s/fits$/burn.v15.tbl/; + } else { + $outTable =~ s/fits$/burn.tbl/; + } + my $outTableReal = $ipprc->file_resolve($outTable, 1); + + my $process = 0; + my $previousTableStyle = 0; + + if ($state == 0) { + $process = 1; + } + elsif ($state == -1) { + $process = 1; + } + elsif ($state == -3) { + $process = 1; + } + elsif ($state == -2) { + &my_die("Aborting as $rawImfile has modified pixel data!"); + } + elsif ($state == $burntoolStateGood) { + $process = 0; + $previousTableStyle = 1; + } + elsif ($state == -1 * $burntoolStateGood) { + $process = 0; + $previousTableStyle = -1; + } + else { + $process = 1; + } + if ($rerun) { + $process = 1; + } + if ($REALRUN == 0) { + print "##Pretending to process as instructed!\n"; + $process = 1; + } + if (($process == 1)||($processNext == 1)) { + my $status; + $processNext = 1; + + my $tempfile = new File::Temp ( TEMPLATE => "$file->{exp_name}.XXXX", + DIR => '/tmp/', + UNLINK => !$save_temps, + SUFFIX => '.fits'); + my $tempPixels = $tempfile->filename; + + # extract local copy of thie chip imfile + $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN); + if ($status) { + &my_die("failed on funpack",$exp_id,$class_id); + } + + # Run burntool + if ($previousTable ne '') { + $status = vsystem ("$burntool $tempPixels $previousTable out=$outTableReal tableonly=t persist=t", $REALRUN); + } else { + $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN); + } + if ($status) { + &my_die("failed on burntool",$exp_id,$class_id); + } + &my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal); + + $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN); + $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN); + if ($status) { + &my_die("failed to replicate imfile"); + } + + # update database with new burntool state + $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState", $REALRUN); + if ($status) { + &my_die("failed to update imfile"); + } + + # generate psvideostats + my $psvidOutfile = "$psvidOutput/psvideostats.$class_id.dat"; + $status = vsystem ("$psvideostats $tempPixels $psvidOutfile $file->{exp_name} $class_id", $REALRUN); + if ($status) { + # die or warn? + &my_die("failed to update imfile"); + } + + $previousTableStyle = -1; + + print "\n"; + } + + if ($previousTableStyle == 1) { + $previousTable = "infits=$rawImfileReal"; + } + elsif ($previousTableStyle == -1) { + $previousTable = "in=$outTableReal"; + } + else { + die "previousTableStyle undefined!\n"; + } + +} + +exit 0; + +sub vsystem { + my $command = shift; + my $realrun = shift; + + print "$command\n"; + + my $status = 0; + if ($realrun) { + $status = system ($command); + } + + return $status; +} + +sub my_die { + my $message = shift; + if ($#_ != -1) { + my $exp_id = shift; + my $class_id = shift; + vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -3",1); + } + printf STDERR "$message\n"; + exit 1; +} Index: /trunk/tools/eam/rawfix.20230221/src/psvideostats.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/psvideostats.sh (revision 42458) +++ /trunk/tools/eam/rawfix.20230221/src/psvideostats.sh (revision 42458) @@ -0,0 +1,240 @@ +#!/usr/bin/env mana + +# psvideostats.sh (input.fits) (output.txt) +# if no video table in this chip, no output + +macro psvideostats.imfile + if ($0 != 5) + echo "USAGE: psvideostats.imfile (input) (output) (expname) (chipID)" + break + end + + local input output expname chipID + $input = $1 + $output = $2 + $expname = $3 + $chipID = $4 + + $HAS_VIDEO = 0 + + # define the header + echo "trying $input" + get.vidstats.imfile $input + + if (not($HAS_VIDEO)) return + + # chip & exposure info + keyword hdr FILENAME filename + keyword hdr FPPOS fppos + + output $output + echo "$expname $chipID $filename $fppos $VIDSTATS" + output stdout +end + +macro get.vidstats.imfile + if ($0 != 2) + echo "USAGE: get.vidstats.imfile (file)" + break + end + + local filename + $filename = $1 + + init.vidcells + + # to find the video cell, we can look at the PHU and grab CELLMODE + rd hdr $filename + keyword hdr CELLMODE cellmode + strstr -q $cellmode V -start Nstart -length Nchar + if ($Nstart == -1) + echo "no video data for this chip" + return + end + + sprintf extname "video_table_xy%02d" cellXY[$Nstart] + + # read the video cell table (need cnpix1,2 to measure flat-corrected flux) + data $filename + break -auto off + read -fits $extname frame_start_time frame_complete_time centroid_x centroid_y fwhm_x fwhm_y flux cnpix1 cnpix2 + if (not($STATUS)) + echo "missing FITS table field for $filename" + return + end + break -auto on + + $HAS_VIDEO = 1 + + # set default values for outputs + init.values + + # calculate the time offset + set dtime = frame_start_time - frame_start_time[0] + set dx = centroid_x - cnpix1 + set dy = centroid_y - cnpix2 + + # a good star measurement is within the window 15 < (dx,dy) 85 pixels . + + set good = (dtime > 4) && (dx > 15) && (dx < 85) && (dy > 15) && (dy < 85) && (dtime < dtime[-1] - 2.0) + subset dts = dtime where good + subset dxs = dx where good + subset dys = dy where good + subset flux_s = flux where good + $GOODFRAC = dxs[] / dx[] + + if (dxs[] == 0) + save.values + return + end + + # generate percentiles: (5, 16, 50, 84, 95) for dxs, dys + get.percentiles dxs; $DX_SIG_RAW = $dPCT_68 / 2.0; $DX_D90_RAW = $dPCT_90; $DX_MED_RAW = $PCT_50 + get.percentiles dys; $DY_SIG_RAW = $dPCT_68 / 2.0; $DY_D90_RAW = $dPCT_90; $DY_MED_RAW = $PCT_50 + vstat -q flux_s; $FLUX_MEDIAN = $MEDIAN; $FLUX_STDEV = $SIGMA + + # smoothed version of dxs, dys: + vmedfilt dxs dxsmooth 10 + vmedfilt dys dysmooth 10 + set seqsm = ramp(dxsmooth) + + # smoothed vector stdev + get.percentiles dxsmooth; $DX_SIG_SMT = $dPCT_68 / 2.0; $DX_D90_SMT = $dPCT_90; $DX_MED_SMT = $PCT_50 + get.percentiles dysmooth; $DY_SIG_SMT = $dPCT_68 / 2.0; $DY_D90_SMT = $dPCT_90; $DY_MED_SMT = $PCT_50 + + # measure long-term drift + vstat -q dxsmooth where (seqsm < 10); $DX_DRIFT_S = $MEDIAN + vstat -q dxsmooth where (seqsm > seqsm[-1] - 10); $DX_DRIFT_E = $MEDIAN + $DX_DRIFT = $DX_DRIFT_E - $DX_DRIFT_S + + vstat -q dysmooth where (seqsm < 10); $DY_DRIFT_S = $MEDIAN + vstat -q dysmooth where (seqsm > seqsm[-1] - 10); $DY_DRIFT_E = $MEDIAN + $DY_DRIFT = $DY_DRIFT_E - $DY_DRIFT_S + + # scatter out of min range + vstat -q dxsmooth where (abs(dxsmooth - $DX_MED_SMT) > 2.0); $FX_OUT_SMT = $NPIX / dxsmooth[] + vstat -q dysmooth where (abs(dysmooth - $DY_MED_SMT) > 2.0); $FY_OUT_SMT = $NPIX / dysmooth[] + + save.values +end + +#### Utilities #### + +macro init.values + $GOODFRAC = 0 + $FLUX_MEDIAN = NAN + $FLUX_STDEV = NAN + $DX_SIG_RAW = NAN + $DY_SIG_RAW = NAN + $DX_D90_RAW = NAN + $DY_D90_RAW = NAN + $DX_SIG_SMT = NAN + $DY_SIG_SMT = NAN + $DX_D90_SMT = NAN + $DY_D90_SMT = NAN + $DX_DRIFT = NAN + $DY_DRIFT = NAN + $FX_OUT_SMT = NAN + $FY_OUT_SMT = NAN +end + +# output values are all Global +macro save.values + sprintf VIDSTAT1 "%5.3f %9.1f %9.1f" $GOODFRAC $FLUX_MEDIAN $FLUX_STDEV + sprintf VIDSTAT2 "%6.2f %6.2f : %6.2f %6.2f" $DX_SIG_RAW $DY_SIG_RAW $DX_D90_RAW $DY_D90_RAW + sprintf VIDSTAT3 "%6.2f %6.2f : %6.2f %6.2f" $DX_SIG_SMT $DY_SIG_SMT $DX_D90_SMT $DY_D90_SMT + sprintf VIDSTAT4 "%6.2f %6.2f : %6.4f %6.4f" $DX_DRIFT $DY_DRIFT $FX_OUT_SMT $FY_OUT_SMT + sprintf VIDSTATS "%s : %s : %s : %s" "$VIDSTAT1" "$VIDSTAT2" "$VIDSTAT3" "$VIDSTAT4" +end + +# only used if needed +macro init.vidcells + if (cellXY?[]) return + + local ix iy cellXY + + # create a lookup table sequence -> xy + delete -q cellXY + for iy 0 8 + for ix 0 8 + $cellXY = $ix\$iy + concat $cellXY cellXY + end + end +end + +# set minRange - maxRange as 2 x (0.05 to 0.95) range +macro get.threshrange + if ($0 != 2) + echo "USAGE: get.threshrange (vector)" + break + end + + local vecname + $vecname = $1 + + vstat -q $vecname + histogram $vecname _tmpNx 0 {2*$MEDIAN} {0.01*$MEDIAN} -range _tmpdx + cumulative _tmpNx _tmpSx + set _tmpFx = _tmpSx / _tmpSx[-1] + threshold -saturate -q _tmpdx _tmpFx 0.05 + $minT = $threshold + + threshold -saturate -q _tmpdx _tmpFx 0.95 + $maxT = $threshold + + $allRange = $maxT - $minT + $midRange = 0.5*($maxT + $minT) + $minRange = $midRange - $allRange + $maxRange = $midRange + $allRange +end + +# set minRange - maxRange as 2 x (0.05 to 0.95) range +macro get.percentiles + if ($0 != 2) + echo "USAGE: get.percentiles (vector)" + break + end + + local vecname + $vecname = $1 + + # percentiles to calculate: + # 16, 84 (1-sigma) + # 5, 95 (2-sigma) + # 50 (median) + vstat -q $vecname + if ($MIN == $MAX) + $PCT_05 = $MIN + $PCT_16 = $MIN + $PCT_50 = $MIN + $PCT_84 = $MIN + $PCT_95 = $MIN + else + + histogram $vecname _tmpNx $MIN $MAX {($MAX - $MIN) / 1000} -range _tmpdx + cumulative _tmpNx _tmpSx + set _tmpFx = _tmpSx / _tmpSx[-1] + + threshold -saturate -q _tmpdx _tmpFx 0.05; $PCT_05 = $threshold + threshold -saturate -q _tmpdx _tmpFx 0.16; $PCT_16 = $threshold + threshold -saturate -q _tmpdx _tmpFx 0.50; $PCT_50 = $threshold + threshold -saturate -q _tmpdx _tmpFx 0.84; $PCT_84 = $threshold + threshold -saturate -q _tmpdx _tmpFx 0.95; $PCT_95 = $threshold + end + + $dPCT_90 = $PCT_95 - $PCT_05 + $dPCT_68 = $PCT_84 - $PCT_16 +end + +#### run as script +if ($SCRIPT) + if ($argv:n != 4) + echo "USAGE: psvideostats.sh (input) (output) (expname) (chipID)" + exit 2 + end + + psvideostats.imfile $argv:0 $argv:1 $argv:2 $argv:3 + exit 0 +end + Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.burntool.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/rawfix.burntool.pt (revision 42458) +++ /trunk/tools/eam/rawfix.20230221/src/rawfix.burntool.pt (revision 42458) @@ -0,0 +1,287 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +if (not($?RAWFIX_BURNTOOL_INIT)) + book init burntool.book + book init burnhost.book + $RAWFIX_BURNTOOL_INIT = 1 +end + +$MAX_BURNHOST = 2 + +# select nights in state burntool.new +# output: burntool.book +task nights.burntool.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/burntool.load.stdout.log + stderr $LOGDIR/burntool.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command check.rawfix.sh state burntool.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout burntool.book + book delpage burntool.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# output: burnhost.book +task nights.burntool.loadhosts + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host local + npending 1 + # note: the tasks which return their results via stdout + # seem to fail if run remotely. + + stdout $LOGDIR/burntool.loadhosts.stdout.log + stderr $LOGDIR/burntool.loadhosts.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages burntool.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in burntool.book (pantaskState == INIT) + book getpage burntool.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword burntool.book $pageName pantaskState RUN + book getword burntool.book $pageName dateobs -var DATEOBS + book getword burntool.book $pageName iter -var ITER + option $pageName + # save the page so we can use it below + + # write an entry point in the log for this night + sprintf tmpline "%s/%s/log.burntool.runhosts.v%d" $mypath $DATEOBS $ITER + date -var mydate + exec echo "# start runhosts $mydate" >> $tmpline + + # the command returns these fields: dateobs stage fixhost chipID + # NOTE: stage == ITER + command get_hosts_burntool.sh $mypath $DATEOBS $ITER + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # queueprint stdout + queuesize stdout -var Nhosts + queue2book -raw-columns -key dateobs:chipID -uniq -setword pantaskState INIT stdout burnhost.book + book setword burntool.book $options:0 NHOST {$Nhosts - 1} + book setword burntool.book $options:0 NDONE 0 + # book list + # book listbook burntool.book + # use the queuesize (Nhosts) to track how many jobs are done / to be done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword burntool.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword burntool.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword burntool.book $options:0 pantasksQuality TIMEOUT + end +end + +task nights.burntool.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 1800 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages burnhost.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in burnhost.book (pantaskState == INIT) + book getpage burnhost.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword burnhost.book $pageName pantaskState RUN + book getword burnhost.book $pageName dateobs -var DATEOBS + book getword burnhost.book $pageName iter -var ITER + book getword burnhost.book $pageName fixhost -var BURNHOST + book getword burnhost.book $pageName chipID -var CHIP_ID + option $pageName $DATEOBS + + stdout $DATEOBS/log.burntool.run.$CHIP_ID.v$ITER + stderr $DATEOBS/log.burntool.run.$CHIP_ID.v$ITER + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf STARTDATE "%s/%s/%s" $YEAR $MONTH $DAY + sprintf ENDDATE "%s/%s/%s,23:59:59" $YEAR $MONTH $DAY + + host -required $BURNHOST + ## check if this host needs to be launched + if (not($?host:$BURNHOST)) + echo "host not defined, creating : $BURNHOST" + $host:$BURNHOST = 0 + end + if ($host:$BURNHOST < $MAX_BURNHOST) + control host add $BURNHOST + $host:$BURNHOST ++ + end + + sprintf psvidOutput "%s/%s" $mypath $DATEOBS + + # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs + periods -exec 0.05 + command ipp_apply_burntool_psvideo.pl --dateobs_begin $STARTDATE --dateobs_end $ENDDATE --class_id $CHIP_ID --dbname gpc1 --psvidOutput $psvidOutput + end + + task.exit 0 + nights.burntool.exit OK + end + + # exit values other than 0: + task.exit default + nights.burntool.exit FAIL + showcommand failure + end + + task.exit crash + nights.burntool.exit CRASH + showcommand crash + end + + # operation times out? + task.exit timeout + nights.burntool.exit TIMEOUT + showcommand timeout + end +end + +# output: burnhost.book +task nights.burntool.clean + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/burntool.clean.stdout.log + stderr $LOGDIR/burntool.clean.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages burntool.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for completed nights in burntool.book (pantaskState == STOP) + book getpage burntool.book 0 -var pageName -key pantaskState STOP + if ("$pageName" == "NULL") break + + # book listpage burntool.book $pageName + + # book setword burntool.book $pageName pantaskState RUN + book getword burntool.book $pageName dateobs -var DATEOBS + book getword burntool.book $pageName pantasksQuality -var QUALITY + + option $pageName + # save the page so we can use it below + + if ("$QUALITY" == "OK") + command update.rawfix.sh $DATEOBS burntool.done + else + command update.rawfix.sh $DATEOBS burntool.fail + end + end + + task.exit 0 + # now that the entry is set to done in the db, remove the pantasks book entry + echo "cleaning this page: $options:0" + book setword burntool.book $options:0 pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword burntool.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword burntool.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword burntool.book $options:0 pantasksQuality TIMEOUT + end +end + +######################### UTILITIES ################################# + +macro nights.burntool.exit + if ($0 != 2) + echo "USAGE: nights.burntool.exit (quality)" + break + end + + book setword burnhost.book $options:0 pantaskState DONE + book getword burntool.book $options:1 NHOST -var Nhosts + book getword burntool.book $options:1 NDONE -var Ndone + book getword burntool.book $options:1 stage -var ITER + $Ndone ++ + book setword burntool.book $options:1 NDONE $Ndone + if ("$1" != "OK") + book setword burntool.book $options:1 pantasksQuality $1 + sprintf tmpline "%s/%s/log.burntool.runhosts.v%d" $mypath $options:1 $ITER + exec echo "failure $1 $options:0" >> $tmpline + else + exec echo "success $1 $options:0" >> $tmpline + end + if ($Ndone == $Nhosts) + book setword burntool.book $options:1 pantaskState STOP + end + book delpage burnhost.book $options:0 +end +