Changeset 42749

Timestamp:

Dec 6, 2024, 1:25:39 PM (20 months ago)

Author:

eugene

Message:

merge changes from trunk : add file.magic and rawfix tools

Location:

branches/eam_branches/ipp-20230313/tools

Files:

• . (modified) (1 prop)
• eam/file.magic (copied) (copied from trunk/tools/eam/file.magic )
• eam/rawfix.20230221/src.v0 (deleted)
• eam/rawfix.20230221/src/get_hosts_fixchip.sh (modified) (1 diff)
• eam/rawfix.20230221/src/getvidstats.sh (copied) (copied from trunk/tools/eam/rawfix.20230221/src/getvidstats.sh )
• eam/rawfix.20230221/src/ipp_apply_burntool_psvideo.pl (modified) (6 diffs)
• eam/rawfix.20230221/src/rawfix.advance.pt (modified) (1 diff)
• eam/rawfix.20230221/src/rawfix.burntool.pt (modified) (1 diff)
• eam/rawfix.20230221/src/update.bnodes.sh (modified) (1 diff)

branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_hosts_fixchip.sh

@@ -27 +27 @@
 
 foreach f ($basedir/$dateobs/XY??.hosts.txt)
-  set myhost = `cat $f | grep -v ippb | sort -k 2n | tail -n 1 | prcol 1`
+# set myhost = `cat $f | grep -v ippb | sort -k 2n | tail -n 1 | prcol 1`
+  set myhost = `awk 'BEGIN{srand()}{print $0, rand()}' $f | grep -v ippb | sort -k 2n | tail -n 5 | sort -k 3n | tail -n 1 | prcol 1`
   set chipID = `echo $f | awk -F/ '{print $NF}' | awk -F. '{print $1}'`
   if ("$myhost" == "") then

branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/ipp_apply_burntool_psvideo.pl

@@ -15 +15 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
+
+use Sys::Hostname;
+my $host = hostname();
+my $date = `date`;
+print "\n\n";
+print "Starting script $0 on $host at $date\n\n";
 
 my ( $class_id, $dateobs_begin, $dateobs_end, $convert, $dbname, $logfile, $verbose, $save_temps, $rerun, $psvidOutput);
@@ -32 +38 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
-          -msg => "Required options: --class_id --dateobs_begin --dateobs_end --dbname",
+          -msg => "Required options: --class_id --dateobs_begin --dateobs_end --dbname --psvidOutput",
           -exitval => 3,
           ) unless
@@ -38 +44 @@
     defined $dateobs_begin and
     defined $dateobs_end and
+    defined $psvidOutput and
     defined $dbname;
 
@@ -65 +72 @@
 $ipprc->redirect_output($logfile) if $logfile;
 
-# Determine the value of a "good" burntool run.
-# XXX this program should always generate the latest version
-# but the processing system must be told to use an earlier version 
-# until the newer version is validated.  Perhaps split these.
-my $config_cmd = "$ppConfigDump -camera $camera -get-key BURNTOOL.STATE.GOOD";
-my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-    run ( command => $config_cmd, verbose => $verbose);
-unless ($success) {
-    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
-}
-
-my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
-    &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
-
-my $burntoolStateGood = 999;
-foreach my $cfg (@$recipeData) {
-    if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') {
-        $burntoolStateGood = $cfg->{value};
-    }
-}
-if ($burntoolStateGood == 999) {
-    &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR);
+my $burntoolStateGood = 15;
+if (0) { 
+    # Determine the value of a "good" burntool run.
+    # XXX this program should always generate the latest version
+    # but the processing system must be told to use an earlier version 
+    # until the newer version is validated.  Perhaps split these.
+
+    my $config_cmd = "$ppConfigDump -camera $camera -get-key BURNTOOL.STATE.GOOD";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $config_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
+    }
+
+    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
+
+    my $burntoolStateGood = 999;
+    foreach my $cfg (@$recipeData) {
+        if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') {
+            $burntoolStateGood = $cfg->{value};
+        }
+    }
+    if ($burntoolStateGood == 999) {
+        &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR);
+    }
 }
 my $outState = -1 * abs($burntoolStateGood);
 
 print "Target burntool version: $burntoolStateGood\n";
+if ($burntoolStateGood != 15) { &my_die ("burntool state is not set to 15\n"); }
 
 # Define list of images to examine.
@@ -101 +113 @@
 $command .= " -dbname $dbname" if defined $dbname;
 
-( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     run ( command => $command, verbose => $verbose);
 unless ($success) {
@@ -151 +163 @@
     my $outTable  = $file->{uri};
     
+    my $data_state = $file->{data_state};
+    if ($data_state eq "lossy") {
+        print "## skipping $rawImfile : data_state $data_state\n";
+        next;
+    }
+
     if($burntoolStateGood == 15) {
         $outTable =~ s/fits$/burn.v15.tbl/;         

branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.advance.pt

@@ -1 +1 @@
 
-$MAX_ACTIVE_NIGHTS = 10
+$MAX_ACTIVE_NIGHTS = 30
 
 ## regularly run the command advance.rawfits.sh

branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.burntool.pt

@@ -160 +160 @@
     stderr $mypath/$YEAR/$DATEOBS/log.burntool.run.$CHIP_ID.v$ITER
 
-    host -required $BURNHOST
+   
+    host $BURNHOST
+#   host -required $BURNHOST
+#   too many jobs wanted the same machines so the processing was clogged.
+#   allow them to go elsewhere if needed.
+    
     ## check if this host needs to be launched
     if (not($?host:$BURNHOST)) 

branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/update.bnodes.sh

@@ -1 +1 @@
 #!/bin/csh
+
+echo "this program is deprecated since we use a directory on b-node for the analysis"
+exit 2
 
 if ($#argv != 2) then