Changeset 30049 for trunk/ippScripts

Timestamp:

Dec 15, 2010, 1:29:28 PM (16 years ago)

Author:

watersc1

Message:

Merge of realtime burntool and new nightly science code into trunk.

Location:

trunk

Files:

• . (modified) (1 prop)
• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/MANIFEST (modified) (1 diff)
• ippScripts/scripts/ipp_apply_burntool_single.pl (copied) (copied from branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl )
• ippScripts/scripts/nightly_science.pl (copied) (copied from branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl )
• ippScripts/scripts/register_exp.pl (modified) (4 diffs)
• ippScripts/scripts/register_imfile.pl (modified) (7 diffs)

trunk/ippScripts/Build.PL

@@ -105 +105 @@
         scripts/whichimage
         scripts/automate_stacks.pl
+        scripts/nightly_science.pl
         scripts/lossy_compress_imfile.pl
         scripts/ipp_apply_burntool.pl
+        scripts/ipp_apply_burntool_single.pl
         scripts/make_burntool_pcontrol.pl
         scripts/bundle_detrends.pl

trunk/ippScripts/MANIFEST

@@ -37 +37 @@
 scripts/ds9_cmf_regions.pl
 scripts/automate_stacks.pl
+scripts/nightly_science.pl
 scripts/lossy_compress_imfile.pl
 scripts/ipp_apply_burntool.pl
+scripts/ipp_apply_burntool_single.pl
 scripts/make_burntool_pcontrol.pl
 scripts/bundle_detrends.pl

trunk/ippScripts/scripts/register_exp.pl

@@ -30 +30 @@
 my $regtool = can_run('regtool') or (warn "can't find regtool" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 if ($missing_tools) {
     warn ("Can't find required tools");
@@ -118 +119 @@
 if (uc($exp_type) eq "NULL") { &my_die ("exp_type  not found", $exp_id, $PS_EXIT_CONFIG_ERROR); }
 
+my ($data_group,$dist_group,$chip_workdir,$reduction);
+($label,$data_group,$dist_group,$end_stage,
+ $tess_id,$chip_workdir,$reduction) = advance_decisions(&value_for_flag($cmdflags,"-dateobs"),
+                                                        &value_for_flag($cmdflags,"-exp_type"),
+                                                        &value_for_flag($cmdflags,"-obs_mode"),
+                                                        &value_for_flag($cmdflags,"-object"),
+                                                        &value_for_flag($cmdflags,"-comment"),
+                                                        $dbname);
 my $command = "$regtool -addprocessedexp";
 $command .= " -exp_id $exp_id";
@@ -127 +136 @@
 $command .= " -end_stage $end_stage" if defined $end_stage;
 $command .= " -tess_id   $tess_id"   if defined $tess_id;
+$command .= " -data_group $data_group" if defined $data_group;
+$command .= " -dist_group $dist_group" if defined $dist_group;
+$command .= " -chip_workdir $chip_workdir" if defined $chip_workdir;
+$command .= " -reduction  $reduction"  if defined $reduction;
 $command .= " -state full";
 $command .= " $cmdflags";
@@ -156 +169 @@
 
 ### Pau.
+sub advance_decisions {
+    my $dateobs = shift;
+    my $exptype = shift;
+    my $obsmode = shift;
+    my $object  = shift;
+    my $comment = shift;
+    my $dbname  = shift;
+
+    # The strings come pre-quoted.
+    $dateobs =~ s/\'//g;
+    $exptype =~ s/\'//g;
+    $obsmode =~ s/\'//g;
+    $object =~ s/\'//g;
+    $comment =~ s/\'//g;
+
+    print "Inside decision engine\n";
+    my ($label,$data_group,$dist_group,$end_stage,$tess_id,$chip_workdir,$reduction);
+    my $target;
+
+    if ($exptype ne 'OBJECT') {
+        print "Aborting because exptype doesn't claim to be OBJECT: $exptype\n";
+        return(undef,undef,undef,"reg",undef,undef,undef);
+    }
+    if ($obsmode eq 'ENGINEERING') {
+        print "Aborting because we don't queue engineering data to chip\n";
+        return(undef,undef,undef,"reg",undef,undef,undef);
+    }
+    # Grab the configuration data.
+    my %nightlyscience_config = ();
+    my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $conf_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR);
+    }
+    print "data loaded\n";
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+    foreach my $entry (@{ $metadata }) {
+        if (${ $entry }{name} eq 'TARGETS') {
+            my @target_data = @{ ${ $entry }{value} };
+            my $this_target = '';
+            foreach my $tentry (@target_data) {
+                if (${ $tentry }{name} eq 'NAME') {
+                    $this_target = ${ $tentry }{value};
+                    $nightlyscience_config{$this_target}{DISTRIBUTION} = $this_target;
+                }
+                elsif (${ $tentry }{name} eq 'DISTRIBUTION') {
+                    $nightlyscience_config{$this_target}{DISTRIBUTION} = ${ $tentry }{value};
+                }
+                elsif (${ $tentry }{name} eq 'TESS') {
+                    $nightlyscience_config{$this_target}{TESS_ID} = ${ $tentry }{value};
+                }
+                elsif (${ $tentry }{name} eq 'OBSMODE') {
+                    $nightlyscience_config{$this_target}{OBSMODE} = ${ $tentry }{value};
+                    $nightlyscience_config{$this_target}{OBSMODE} =~ s/%/\.\*/g;
+                }
+                elsif (${ $tentry }{name} eq 'OBJECT') {
+                    $nightlyscience_config{$this_target}{OBJECT} = ${ $tentry }{value};
+                    $nightlyscience_config{$this_target}{OBJECT} =~ s/%/\.\*/g;
+                }
+                elsif (${ $tentry }{name} eq 'COMMENT') {
+                    $nightlyscience_config{$this_target}{COMMENT} = ${ $tentry }{value};
+                    $nightlyscience_config{$this_target}{COMMENT} =~ s/%/\.\*/g;
+                }
+                elsif (${ $tentry }{name} eq 'STACKABLE') {
+                    $nightlyscience_config{$this_target}{STACKABLE} = ${ $tentry }{value};
+                }
+                elsif (${ $tentry }{name} eq 'DIFFABLE') {
+                    $nightlyscience_config{$this_target}{DIFFABLE} = ${ $tentry }{value};
+                }
+                elsif (${ $tentry }{name} eq 'REDUCTION') {
+                    $nightlyscience_config{$this_target}{REDUCTION} = ${ $tentry }{value};
+                }
+            }
+        }
+    }
+    print "data parsed\n";
+    my $found_target;
+    foreach $target (keys %nightlyscience_config) {
+#       print "TEST: $target ($obsmode $object $comment) ($nightlyscience_config{$target}{OBSMODE} $nightlyscience_config{$target}{OBJECT} $nightlyscience_config{$target}{COMMENT}\t";
+        my $match = 0;
+        my $possible = 0;
+        if (exists($nightlyscience_config{$target}{OBSMODE})) {
+            $possible++;
+            if ($obsmode =~ /$nightlyscience_config{$target}{OBSMODE}/) {
+                $match++;
+            }
+        }
+        if (exists($nightlyscience_config{$target}{OBJECT})) {
+            $possible++;
+            if ($object =~ /$nightlyscience_config{$target}{OBJECT}/) {
+                $match++;
+            }
+        }
+        if (exists($nightlyscience_config{$target}{COMMENT})) {
+            $possible++;
+            if ($comment =~ /$nightlyscience_config{$target}{COMMENT}/) {
+                $match++;
+            }
+        }
+        print "$possible $match\n";
+        if (($possible > 0)&&($match == $possible)) {
+            $found_target = $target;
+            last;
+        }
+    }
+    unless ($found_target) {
+        print "No acceptible target found.\n";
+        return(undef,undef,undef,"reg",undef,undef,undef);
+    }
+    print "Found this target: $found_target\n";
+    print "$dateobs\n";
+    $target = $found_target;
+    
+    my $date  = $dateobs;
+    $date =~ s/T.*//;
+    my $workdir_date = $date;
+    $date =~ s/-//g;
+    $workdir_date =~ s%-%/%g;
+    print "$dateobs $date $workdir_date\n";    
+    $label = "${target}.nightlyscience";
+    $data_group = "${target}.${date}";
+    $dist_group = $nightlyscience_config{$target}{DISTRIBUTION};
+    $end_stage  = 'warp';
+    $tess_id    = $nightlyscience_config{$target}{TESS_ID};
+    $chip_workdir = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}/";
+    if (exists($nightlyscience_config{$target}{REDUCTION})) {
+        $reduction  = $nightlyscience_config{$target}{REDUCTION};
+    }
+    else {
+        undef($reduction);
+    }
+
+    
+    return ($label,$data_group,$dist_group,$end_stage,$tess_id,$chip_workdir,$reduction);
+}
+            
+            
+            
 
 sub cache_run

trunk/ippScripts/scripts/register_imfile.pl

@@ -19 +19 @@
 use File::Basename qw( basename);
 use IPC::Run;
-use IPC::Cmd 0.36 qw( can_run );
+use IPC::Cmd 0.36 qw( can_run run );
 use PS::IPP::Config 1.01 qw( :standard );
+use PS::IPP::Metadata::List qw( parse_md_list );
 use PS::IPP::Metadata::Config;
 use Math::Trig;
@@ -35 +36 @@
 my $ppStatsFromMetadata = can_run( 'ppStatsFromMetadata' ) or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
 my $ppConfigDump = can_run( 'ppConfigDump' ) or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+my $ippApplyBurntoolSingle = can_run( 'ipp_apply_burntool_single.pl' ) or (warn "Can't find ipp_apply_burntool_single" and $missing_tools = 1);
 
 my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $bytes, $md5sum, $dbname, $verbose, $no_update, $no_op, $logfile);
@@ -82 +84 @@
 
 my $cmdflags;
+my $burntoolStateGood;
 # Run ppStats on the input file
 {
@@ -114 +117 @@
     my $burntoolState = 0;
     my $isGPC1 = 0;
-    my $burntoolStateGood = 0;
+    $burntoolStateGood = 0;
     foreach my $line (split /\n/, $out1) {
         if ($line =~ /FPA.BURNTOOL.APPLIED/) {
@@ -174 +177 @@
 $command .= " -hostname $host" if defined $host;
 $command .= " -dbname $dbname" if defined $dbname;
-$command .= " -data_state full";
+$command .= " -data_state check_burntool";
 $command .= " $cmdflags";
 
+print "$command\n";
 # determine solar-system parameters
 my $longitude = &value_for_flag($cmdflags, 0.0, "-longitude");
@@ -206 +210 @@
     }
 }
+
+# This might have a race condition
+# $date_end = $dateobs;
+# $date_start = $dateobs - 30 minutes ? dateobs_UTC_midnight?
+# lock file?
+# if exp_type = DARK and date > MIDNIGHT HST { wait }
+# system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log"); 
 
 $now_time = localtime();
@@ -222 +233 @@
     print "skipping command: $command\n";
 }
+
+# We now have an imfile in the database, check if we can burntool it.  If not, continue on.
+
+{ 
+    my $mdcParser  = PS::IPP::Metadata::Config->new;
+
+    my $class_id   = &value_for_flag($cmdflags, 0.0, "-class_id");
+    my $check_date       = &value_for_flag($cmdflags, 0.0, "-dateobs");
+    $check_date =~ s/T.*$//;
+    my $exp_name   = $tmp_exp_name;
+    
+
+    my $bt_check_command = "$regtool -checkburntoolimfile ";
+    $bt_check_command .= " -class_id $class_id ";
+    $bt_check_command .= " -date $check_date ";
+    $bt_check_command .= " -valid_burntool $burntoolStateGood ";
+    $bt_check_command .= " -exp_name $exp_name ";
+    $bt_check_command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $bt_check_command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform regtool: $error_code", 
+                $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR);
+    }
+
+    my @whole = split /\n/, (join "", @$stdout_buf);
+    my @single = ();
+    my $exposures;
+    while ( scalar @whole > 0 ) {
+        my $value = shift @whole;
+        push @single, $value;
+        if ($value =~ /^\s*END\s*$/) {
+            push @single, "\n";
+            
+            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
+            &my_die("Unable to parse output from regtool", 
+                    $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR) unless
+                        defined $list;
+            push @{ $exposures }, @$list;
+            @single = ();
+        }
+    }
+    
+    # We only care about the final entry, as that contains the exposure we are.
+    
+    my $regtool_update = "$regtool -updateprocessedimfile ";
+    $regtool_update .= "-dbname $dbname " if defined $dbname;
+    $regtool_update .= "-exp_id $exp_id -class_id $class_id ";
+
+    my $burntool_data = pop(@{ $exposures });
+    if ($burntool_data->{burnable} == 0) {
+        $regtool_update .= " -burntool_state 0 -set_state pending_burntool ";
+    }
+    else {
+        my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname ";
+        $apply_command .= " --class_id $class_id --exp_id $exp_id ";
+        $apply_command .= " --this_uri $burntool_data->{uri} ";
+        $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri};
+        $apply_command .= " --imfile_state $burntool_data->{imfile_state} ";
+        $apply_command .= " --verbose " if $verbose;
+        print "$apply_command\n";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            IPC::Cmd::run(command => $apply_command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code");
+            exit($error_code);
+        }
+    }    
+
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            warn ("Unable to perform regtool -addprocessedimfile: $error_code");
+            exit($error_code);
+        }
+    } else {
+        print "skipping command: $command\n";
+    }
+
+}
+
+
 
 $now_time = localtime();