Index: /tags/ipp-20101215/ippScripts/scripts/register_imfile.pl =================================================================== --- /tags/ipp-20101215/ippScripts/scripts/register_imfile.pl (revision 30182) +++ /tags/ipp-20101215/ippScripts/scripts/register_imfile.pl (revision 30183) @@ -84,6 +84,6 @@ my $cmdflags; -my $burntoolStateGood; -my $burntoolState; +my $burntoolStateTarget; +my $burntoolStateCurrent; # Run ppStats on the input file { @@ -116,11 +116,11 @@ # Manually parse the burntool_state entry. - $burntoolState = 0; + $burntoolStateCurrent = 0; my $isGPC1 = 0; - $burntoolStateGood = 0; + $burntoolStateTarget = 0; foreach my $line (split /\n/, $out1) { if ($line =~ /FPA.BURNTOOL.APPLIED/) { $line =~ s/^\s+//; - $burntoolState = (split /\s+/, $line)[2]; + $burntoolStateCurrent = (split /\s+/, $line)[2]; } if ($line =~ /FPA.CAMERA/) { @@ -132,8 +132,8 @@ } if ($isGPC1 != 1) { - $burntoolState = 0; # If it's not GPC1, you shouldn't have run burntool. - } - elsif (($isGPC1 == 1) && ($burntoolState == 1)) { -# print STDERR "In the good region: >>$burntoolState<<\n"; + $burntoolStateCurrent = 0; # If it's not GPC1, you shouldn't have run burntool. + } + elsif (($isGPC1 == 1) && ($burntoolStateCurrent == 1)) { +# print STDERR "In the good region: >>$burntoolStateCurrent<<\n"; my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -149,16 +149,18 @@ if ($line =~ /BURNTOOL.STATE.GOOD/) { $line =~ s/^\s+//; - $burntoolStateGood = (split /\s+/, $line)[2]; + $burntoolStateTarget = (split /\s+/, $line)[2]; last; } } - $burntoolState = $burntoolStateGood; # Positive because this has the header table. - - } - $cmdflags .= " -burntool_state $burntoolState "; + # XXX why was this being equated?? + # $burntoolState = $burntoolStateGood; # Positive because this has the header table. + + } + $cmdflags .= " -burntool_state $burntoolStateTarget "; } $now_time = localtime(); printf STDERR "\ndone with ppStats: %s\n", $now_time if $verbose; +printf STDERR "\nburntool state current: %d target: %d\n", $burntoolStateCurrent, $burntoolStateTarget; # we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type @@ -178,5 +180,5 @@ $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; -if (abs($burntoolState) == $burntoolStateGood) { +if (abs($burntoolStateCurrent) == $burntoolStateTarget) { $command .= " -data_state full"; } @@ -222,5 +224,5 @@ # lock file? # if exp_type = DARK and date > MIDNIGHT HST { wait } -# system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log"); +# system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log"); $now_time = localtime(); @@ -242,5 +244,5 @@ # We now have an imfile in the database, check if we can burntool it. If not, continue on. -if (abs($burntoolState) != $burntoolStateGood) { +if (abs($burntoolStateCurrent) != $burntoolStateTarget) { my $mdcParser = PS::IPP::Metadata::Config->new; @@ -249,19 +251,19 @@ $check_date =~ s/T.*$//; my $exp_name = $tmp_exp_name; - + my $bt_check_command = "$regtool -checkburntoolimfile "; $bt_check_command .= " -class_id $class_id "; $bt_check_command .= " -date $check_date "; - $bt_check_command .= " -valid_burntool $burntoolStateGood "; + $bt_check_command .= " -valid_burntool $burntoolStateTarget "; $bt_check_command .= " -exp_name $exp_name "; $bt_check_command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $bt_check_command, verbose => $verbose); + run ( command => $bt_check_command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die_for_update("Unable to perform regtool: $error_code", - $exp_id, $exp_name, $class_id, $PS_EXIT_SYS_ERROR); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die_for_update("Unable to perform regtool: $error_code", + $exp_id, $exp_name, $class_id, $PS_EXIT_SYS_ERROR); } @@ -270,20 +272,20 @@ my $exposures; while ( scalar @whole > 0 ) { - my $value = shift @whole; - push @single, $value; - if ($value =~ /^\s*END\s*$/) { - push @single, "\n"; - - my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ); - &my_die_for_update("Unable to parse output from regtool", - $exp_id, $exp_name, $class_id, $PS_EXIT_SYS_ERROR) unless - defined $list; - push @{ $exposures }, @$list; - @single = (); - } - } - + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ); + &my_die_for_update("Unable to parse output from regtool", + $exp_id, $exp_name, $class_id, $PS_EXIT_SYS_ERROR) unless + defined $list; + push @{ $exposures }, @$list; + @single = (); + } + } + # We only care about the final entry, as that contains the exposure we are. - + my $regtool_update = "$regtool -updateprocessedimfile "; $regtool_update .= "-dbname $dbname " if defined $dbname; @@ -292,35 +294,35 @@ my $burntool_data = pop(@{ $exposures }); if ($burntool_data->{burnable} == 0) { - $regtool_update .= " -burntool_state 0 -set_state pending_burntool "; - unless ($no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - IPC::Cmd::run(command => $regtool_update, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn ("Unable to perform regtool -addprocessedimfile: $error_code"); - exit($error_code); - } - } else { - print "skipping command: $command\n"; - } + $regtool_update .= " -burntool_state 0 -set_state pending_burntool "; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + IPC::Cmd::run(command => $regtool_update, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } + } else { + print "skipping command: $command\n"; + } } else { - my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname "; - $apply_command .= " --class_id $class_id --exp_id $exp_id "; - $apply_command .= " --this_uri $burntool_data->{uri} "; - $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri}; - $apply_command .= " --imfile_state $burntool_data->{imfile_state} "; - $apply_command .= " --verbose " if $verbose; - print "$apply_command\n"; - unless ($no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - IPC::Cmd::run(command => $apply_command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code"); - exit($error_code); - } - } - } + my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname "; + $apply_command .= " --class_id $class_id --exp_id $exp_id "; + $apply_command .= " --this_uri $burntool_data->{uri} "; + $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri}; + $apply_command .= " --imfile_state $burntool_data->{imfile_state} "; + $apply_command .= " --verbose " if $verbose; + print "$apply_command\n"; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + IPC::Cmd::run(command => $apply_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code"); + exit($error_code); + } + } + } Index: /tags/ipp-20101215/ippTasks/nightly_science.pro =================================================================== --- /tags/ipp-20101215/ippTasks/nightly_science.pro (revision 30182) +++ /tags/ipp-20101215/ippTasks/nightly_science.pro (revision 30183) @@ -132,4 +132,9 @@ # macro ns.add.date + if ($0 != 2) + echo "USAGE: ns.add.date (YYYY-MM-DD)" + break + end + book newpage nsData $1 book setword nsData $1 nsState NEW Index: /tags/ipp-20101215/ippTasks/register.pro =================================================================== --- /tags/ipp-20101215/ippTasks/register.pro (revision 30182) +++ /tags/ipp-20101215/ippTasks/register.pro (revision 30183) @@ -79,4 +79,22 @@ macro register.revert.off task register.imfile.revert + active false + end +end + +macro burntool.on + task register.burntool.load + active true + end + task register.burntool.run + active true + end +end + +macro burntool.off + task register.burntool.load + active false + end + task register.burntool.run active false end @@ -529,5 +547,5 @@ $today = `date -u +%Y-%m-%d` # debugging purposes -# $today = "2010-12-06" +# $today = "2010-12-25" $run = $run -date $today -valid_burntool $valid_burntool_value if ($DB:n == 0) Index: /tags/ipp-20101215/ippTools/share/regtool_pendingburntoolimfile.sql =================================================================== --- /tags/ipp-20101215/ippTools/share/regtool_pendingburntoolimfile.sql (revision 30182) +++ /tags/ipp-20101215/ippTools/share/regtool_pendingburntoolimfile.sql (revision 30183) @@ -23,9 +23,9 @@ LEFT JOIN pzDownloadExp USING(exp_name) LEFT JOIN newExp ON exp_name = tmp_exp_name - LEFT JOIN newImfile USING(exp_id) + LEFT JOIN newImfile USING(exp_id) LEFT JOIN rawExp USING(exp_id, exp_name) LEFT JOIN rawImfile ON (rawImfile.exp_name = newExp.tmp_exp_name AND rawImfile.tmp_class_id = newImfile.tmp_class_id) WHERE summitExp.dateobs >= '@DATE@T00:00:00' - AND summitExp.dateobs <= '@DATE@T23:59:59' + AND summitExp.dateobs <= '@DATE@T23:59:59' ORDER BY newImfile.tmp_class_id,summitExp.dateobs Index: /tags/ipp-20101215/ippTools/src/regtool.c =================================================================== --- /tags/ipp-20101215/ippTools/src/regtool.c (revision 30182) +++ /tags/ipp-20101215/ippTools/src/regtool.c (revision 30183) @@ -74,6 +74,6 @@ // imfile MODECASE(REGTOOL_MODE_PENDINGIMFILE, pendingimfileMode); - MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileMode); - MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode); + MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileMode); + MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode); MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); MODECASE(REGTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); @@ -201,5 +201,5 @@ psFree(query); query = rep; - + psStringSubstitute(&query,exp_name,"@EXP_NAME@"); psStringSubstitute(&query,class_id,"@CLASS_ID@"); @@ -215,5 +215,5 @@ } psFree(query); - + psArray *output = p_psDBFetchResult(config->dbh); if (!output) { @@ -227,5 +227,5 @@ psError(PXTOOLS_ERR_PROG, false, "unknown error"); } - + return false; } @@ -243,5 +243,5 @@ if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| - (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { + (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { ok_to_burn = false; } @@ -255,5 +255,5 @@ already_burned = false; } - + if (previous_uri) { psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); @@ -264,5 +264,5 @@ previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); } - + // negate simple so the default is true if (!ippdbPrintMetadatas(stdout, output, "regBurntoolImfile", !simple)) { @@ -271,10 +271,10 @@ return false; } - + psFree(output); - + return(true); - + } @@ -297,8 +297,8 @@ psFree(query); query = rep; - + psStringSubstitute(&query,date,"@DATE@"); - // fprintf(stderr,"%s",query); + // fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { @@ -309,5 +309,5 @@ } psFree(query); - + psArray *output = p_psDBFetchResult(config->dbh); if (!output) { @@ -321,7 +321,9 @@ psError(PXTOOLS_ERR_PROG, false, "unknown error"); } - + return false; } + + // fprintf (stderr, "found %ld rows\n", output->n); if (!psArrayLength(output)) { psTrace("regtool", PS_LOG_INFO, "no rows found"); @@ -337,5 +339,5 @@ psString this_class_id = NULL; psArray *results = psArrayAllocEmpty(60); // List of suggested imfiles to burntool. - + for (long i = 0; i < output->n; i++) { psMetadata *row = output->data[i]; @@ -356,7 +358,7 @@ if (previous_class_id) { if (strcmp(this_class_id,previous_class_id) != 0) { - ok_to_burn = true; - already_burned = true; - previous_uri = NULL; + ok_to_burn = true; + already_burned = true; + previous_uri = NULL; } } @@ -364,5 +366,5 @@ // Write the URIs as well. this_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Duplicate, but helpful for my debugging. - psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri); + psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri); if (previous_uri) { psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); @@ -372,11 +374,11 @@ // Convert bits of the SQL query into booleans describing the data state if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| - (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { - // printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id); - ok_to_burn = false; + (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { + // printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id); + ok_to_burn = false; } if (already_burned == false) { - // printf("already_burned looks false %s %s\n",this_uri,this_class_id); - ok_to_burn = false; + // printf("already_burned looks false %s %s\n",this_uri,this_class_id); + ok_to_burn = false; } if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) { @@ -391,44 +393,44 @@ // Check the uri for this exposure if (!this_uri) { - ok_to_burn = false; - already_burned = false; - - // Save this round for next round. - psFree(previous_class_id); - psFree(previous_uri); - previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); - psStringSubstitute(&previous_class_id,"XY","ota"); - previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. - continue; - } - - // printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded")); - - // If the state of this imfile is not "pending_burntool" then we can't burn it. + ok_to_burn = false; + already_burned = false; + + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; + } + + // printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded")); + + // If the state of this imfile is not "pending_burntool" then we can't burn it. if (!ignore_state) { psString imfile_state = psMetadataLookupStr(NULL,row,"imfile_state"); if (!imfile_state) { // imfile state is NULL, so we probably aren't registered. - ok_to_burn = false; - - // Save this round for next round. - psFree(previous_class_id); - psFree(previous_uri); - previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); - psStringSubstitute(&previous_class_id,"XY","ota"); - previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. - continue; + ok_to_burn = false; + + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; } - if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states - // Save this round for next round. - psFree(previous_class_id); - psFree(previous_uri); - previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); - psStringSubstitute(&previous_class_id,"XY","ota"); - previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. - continue; + if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; } } - + // Determine if we've already suggested an entry for this ota, and if not, copy this // suggestion into our output result list. @@ -439,5 +441,5 @@ previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); psStringSubstitute(&previous_class_id,"XY","ota"); - previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. continue; } @@ -450,7 +452,7 @@ previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); psStringSubstitute(&previous_class_id,"XY","ota"); - previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. - } - + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + } + // negate simple so the default is true if (!ippdbPrintMetadatas(stdout, results, "regPendingBurntoolImfile", !simple)) { @@ -460,14 +462,14 @@ return false; } - + psFree(output); psFree(results); - + return(true); - -} - - + +} + + static bool addprocessedimfileMode(pxConfig *config) @@ -545,5 +547,5 @@ PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); PXOPT_LOOKUP_BOOL(video_cells, config->args, "-video_cells", false); - + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); @@ -618,5 +620,5 @@ bytes, md5sum, - video_cells, + video_cells, 0 // burntool_state )) { @@ -806,9 +808,9 @@ PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); - if ((fault == INT16_MAX) && !isfinite(burntool_state) && !(set_state)) { + if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state) { psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected"); return false; } - if ((fault != INT16_MAX) && isfinite(burntool_state)) { + if ((fault != INT16_MAX) && (burntool_state != INT16_MAX)) { psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected"); return false; @@ -1312,8 +1314,8 @@ if (!pxchipQueueByExpTag(config, exp_id, - chip_workdir, + chip_workdir, label, - data_group, - dist_group, + data_group, + dist_group, reduction, NULL, // expgroup