Changeset 42400

Timestamp:

Mar 8, 2023, 9:44:22 AM (3 years ago)

Author:

eugene

Message:

updates to rawfix

Location:

trunk/tools/eam/rawfix.20230221/src

Files:

• check.rawfix.sh (modified) (2 diffs)
• check_chip_locations.pl (modified) (7 diffs)
• get_hosts_md5s.sh (added)
• get_md5s_instances.pl (modified) (4 diffs)
• rawfix.pt (added)
• update.bnodes.sh (added)
• update.rawfix.sh (modified) (1 diff)

trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh

@@ -16 +16 @@
 set DBPASS = "dvo"
 
-set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME"
+# set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME"
+set DBOPTS = "-B -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME"
+
+if ($mode == "all") then
+  if ($#argv != 1) then
+    echo "USAGE: check.rawfix.sh all"
+    exit 2
+  endif
+
+  mysql $DBOPTS -e "select dateobs, state from nights"
+  exit 0
+endif
+
+if ($mode == "stats") then
+  if ($#argv != 1) then
+    echo "USAGE: check.rawfix.sh stats"
+    exit 2
+  endif
+
+  mysql $DBOPTS -e "select count(dateobs) Nnights, state from nights group by state"
+  exit 0
+endif
 
 if ($mode == "night") then
@@ -29 +50 @@
 endif
 
-if ($mode == "new") then
-  mysql $DBOPTS -e "select dateobs from nights where (state = 'new')"
+if ($mode == "state") then
+  if ($#argv != 2) then
+    echo "USAGE: check.rawfix.sh state (state)"
+    exit 2
+  endif
+
+  set mystate = $2
+
+  if ($mystate == "new") goto good_state
+  if ($mystate == "rsync") goto good_state
+  if ($mystate == "md5sum.new") goto good_state
+
+  echo "ERROR: unknown mode $mystate"
+  exit 2
+
+  good_state:
+  mysql $DBOPTS -e "select dateobs from nights where (state = '$mystate')"
   exit 0
 endif

trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl

@@ -40 +40 @@
 
     # get the list of chip neb paths for one chip
-    my @neb_paths = &get_chip_paths ($fulldate, $chip_id);
+    my ($neb_paths_r, $ds_array_r, $md_array_r) = &get_chip_paths ($fulldate, $chip_id);
+
+    # I think this makes a copy of the arrays, which is inefficient
+    # my @neb_paths = @$neb_paths_r;
+    # my @ds_array  = @$ds_array_r;
+    # my @md_array  = @$md_array_r;
 
     # reset the hostcount for this chip
@@ -46 +51 @@
 
     # find the neb instances for each chip nebulous entry
-    foreach my $nebname (@neb_paths) {
+
+    for (my $i = 0; $i < @$neb_paths_r; $i++) {
+        my $nebname = $neb_paths_r->[$i]; # equivalent to ${$neb_paths_r}[$i]
+        my $data_st = $ds_array_r->[$i];
+        my $md5_sum = $md_array_r->[$i];
+
+        # print STDERR "nebname: $nebname, dstate: $data_st, md5: $md5_sum\n";
 
         # get info for all instances of this chip
@@ -54 +65 @@
 
         # determine the status of the instances
-        my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id);
+        my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum);
 
         &check_instance_state ($outfh, $ext_id, $Nmain, $Nback, $Ndead);
@@ -130 +141 @@
     my $chip_id = $_[1];
 
-    my @nebarray;
-
-    my $sql_chip_paths = "SELECT uri from rawImfile WHERE class_id = '$chip_id' and dateobs > '$fulldate,00:00:00' and dateobs < '$fulldate,23:59:59'";
+    my (@nebarray, @ds_array, @md_array);
+
+    my $sql_chip_paths = "SELECT uri, data_state, md5sum from rawImfile WHERE class_id = '$chip_id' and dateobs > '$fulldate,00:00:00' and dateobs < '$fulldate,23:59:59'";
     if ($DEBUG > 5) { print "sql: $sql_chip_paths\n"; }
 
@@ -145 +156 @@
         # there should only be a single value (only uri selected above)
         my $Nvalues = @values;
-        if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; }
-        if ($Nvalues  > 1) { print "ERROR: too many uri entries?\n"; next; }
+        if ($Nvalues != 3) { print "ERROR: invalid response?\n"; next; }
         
         push (@nebarray, $values[0]);
-        if ($DEBUG > 4) { print "$values[0]\n"; }
-    }
-    
-    return @nebarray;
+        push (@ds_array, $values[1]);
+        push (@md_array, $values[2]);
+        if ($DEBUG > 4) { print "$values[0]\n $values[1]\n $values[2]\n"; }
+    }
+
+    # return references so we can keep the three arrays distinct
+    return (\@nebarray, \@ds_array, \@md_array);
 }
 
@@ -177 +190 @@
 sub check_instance_xattr {
 
-    my $ext_id = $_[1];
-
-    my $rowref = $_[0];
+    my $rowref  = $_[0];
+    my $ext_id  = $_[1];
+    my $data_st = $_[2];
+    my $md5_sum = $_[3];
+
     my @inforows = @$rowref;
     my $Nrows    = @inforows;
@@ -218 +233 @@
         }
         my $fh = $hostfile{$hostname};
-        print $fh "$instance\n";
+        print $fh "$instance $data_st $md5_sum\n";
 
         if ($xattr == 0) { $Nmain ++; }

trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl

@@ -21 +21 @@
 
 # read the list of instances:
-my @ins_list = &load_instance_list ($input);
+my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ($input);
 
 my $output = $input;
@@ -29 +29 @@
 open (OUTFILE, ">$output");
 
-foreach my $instance (@ins_list) {
+# foreach my $instance (@$ins_list_r) {
+for (my $i = 0; $i < @$ins_list_r; $i ++) {
+
+    my $instance   = $ins_list_r->[$i];
+    my $data_state = $dst_list_r->[$i];
+    my $md5sum_tru = $md5_list_r->[$i];
 
     my $filename = $instance;
     $filename =~ s|file://||;
-    # print "filename: $filename\n";
 
-    my $line = `md5sum $filename`;
+    my $line;
+    if (0) {
+        $line = "fakevalue $filename"; # XXX TEST: disable calculation:
+    } else {
+        $line = `md5sum $filename`;
+    }
 
     my ($md5sum, $rawfile) = split (" ", $line);
-    # print "md5sum: $md5sum\n";
-
-    print OUTFILE "$instance $md5sum\n";
+    print OUTFILE "$instance $data_state $md5sum_tru $md5sum\n";
 }
 close OUTFILE;
-
-die "all done";
+exit 0;
 
 # read the list of chip_ids:
@@ -50 +56 @@
     my $infile = $_[0];
 
-    my @ins_list;
+    my (@ins_list, @dst_list, @md5_list);
 
     open(FILE,$infile) or die "cannot open file $infile\n"; 
@@ -61 +67 @@
         my @words = split (/\s+/,$line);
 
-        if (@words != 1) { die "error in input $infile : $line\n"; }
+        if (@words != 3) { die "error in input $infile : $line\n"; }
         push (@ins_list, $words[0]);
+        push (@dst_list, $words[1]);
+        push (@md5_list, $words[2]);
     }
     close (FILE);
 
-    return @ins_list;
+    return (\@ins_list, \@dst_list, \@md5_list);
 }
 

trunk/tools/eam/rawfix.20230221/src/update.rawfix.sh

@@ -2 +2 @@
 
 if ($#argv < 1) then
-  echo "USAGE: update.rawfix.sh (night) (state)"
+  echo "USAGE: update.rawfix.sh (night) (state) [quality]"
   echo "   EG: update.rawfix.sh 20100501 new"
   exit 2