Index: /trunk/tools/eam/rawfix.20230221/src/advance.rawfix.pl
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/advance.rawfix.pl	(revision 42446)
+++ /trunk/tools/eam/rawfix.20230221/src/advance.rawfix.pl	(revision 42446)
@@ -0,0 +1,261 @@
+#! /usr/bin/env perl
+
+my $DEBUG = 2;
+my $time = time();
+
+use DBI;
+use Carp;
+use Pod::Usage qw( pod2usage );
+
+use strict;
+use warnings FATAL => qw( all );
+use Getopt::Long qw( GetOptions :config auto_help auto_version );
+
+# USAGE: advance.rawfix.pl --mode (all/night) --dateobs (night) --topdir (dirname) --Nmax Nmax
+
+my ($mode, $topdir, $fulldate, $dateword, $Nmax) = &parse_cmdopts;
+my ($raw_dbh) = &parse_db_config ("$topdir/nebulous.config");
+
+if ($DEBUG >= 4) {
+    print "mode     : $mode    \n";
+    print "topdir   : $topdir  \n";
+    if (defined $fulldate) { print "fulldate : $fulldate\n"; }
+    if (defined $dateword) { print "dateword : $dateword\n"; }
+    if (defined $Nmax    ) { print "Nmax     : $Nmax    \n"; }
+}
+
+if ($mode eq "all") {
+    &advance_all ($Nmax);
+    exit 0;
+} 
+
+if ($mode eq "night") {
+    &advance_night ($dateword);
+    exit 0;
+} 
+
+die "invalid mode $mode\n";
+
+sub advance_all {
+
+    my $Nmax = $_[0];
+
+    # we allow $Nmax to be currently running
+
+    # first count the number of nights already in state 'STAGE.new'
+    # XXX: use a list of valid .new states?
+    my $sql_get_new = "SELECT dateobs from nights where state like '%.new'";
+    my $newref   = $raw_dbh->selectall_arrayref( $sql_get_new );
+    my @newarray = @$newref;
+    my $Nnew    = @newarray;
+
+    print "$Nnew night(s) already running\n"; 
+
+    if ($Nnew >= $Nmax) { return; }
+
+    # select and advance nights already running
+    my $sql_get_run = "SELECT dateobs from nights where state like '%.done'";
+    my $runref   = $raw_dbh->selectall_arrayref( $sql_get_run );
+    my @runarray = @$runref;
+    my $Nrun    = @runarray;
+
+    print "$Nrun night(s) finished a stage\n"; 
+
+    my $i = 0;
+    my $Ntotal = $Nnew;
+    while (($i < $Nrun) and ($Ntotal < $Nmax)) {
+	my $valref = $runarray[$i];
+	my @values = @$valref;
+  	my $dateobs  = $values[0];
+  
+	my $newstate = advance_night ($dateobs);
+	if ($newstate eq "NEW") { $Ntotal ++; }
+	$i ++;
+	# print "$dateobs : $i $Ntotal : $Nrun $Nmax\n";
+    }
+    if ($Ntotal >= $Nmax) { return; }
+
+    # select and advance nights not yet running
+    $sql_get_run = "SELECT dateobs from nights where state = 'new'";
+    $runref   = $raw_dbh->selectall_arrayref( $sql_get_run );
+    @runarray = @$runref;
+    $Nrun    = @runarray;
+
+    print "$Nrun night(s) ready to start\n"; 
+
+    $i = 0;
+    while (($i < $Nrun) and ($Ntotal < $Nmax)) {
+	my $valref = $runarray[$i];
+	my @values = @$valref;
+  	my $dateobs  = $values[0];
+  
+	my $newstate = advance_night ($dateobs);
+	if ($newstate eq "NEW") { $Ntotal ++; }
+	$i ++;
+    }
+    return;
+}
+
+sub advance_night {
+
+    my $dateobs = $_[0];
+
+    ## get the current state of this night:
+    my $fields = "";
+    $fields .= " Nchip,";
+    $fields .= " Ngood,";
+    $fields .= " Ndead,";
+    $fields .= " Ngone,";
+    $fields .= " Nsing,";
+    $fields .= " Ndups,";
+    $fields .= " Nsbck,";
+    $fields .= " Nxbck,";
+    $fields .= " Ncul1,";
+    $fields .= " Ncul2,";
+    $fields .= " Ncul3,";
+    $fields .= " Nfail";
+
+    my $sql_get_night = "SELECT dateobs, state, iter, $fields from nights where dateobs = '$dateobs'";
+    my $rowref   = $raw_dbh->selectall_arrayref( $sql_get_night );
+    my @tmparray = @$rowref;
+    my $Nrows    = @tmparray;
+    if ($Nrows != 1) { die "invalid response from database for night $dateobs\n"; }
+
+    my $valref = $tmparray[0];
+    my @values = @$valref;
+  
+    my $datenew  = $values[0];
+    my $state    = $values[1];
+    my $iter     = $values[2];
+
+    if ($datenew != $dateobs) { die "inconsistent result from rawfix database for $dateobs (wrong dateobs?)\n"; }
+
+    my $Nchip    = $values[3];
+    my $Ngood    = $values[4];
+    my $Ndead    = $values[5];
+    my $Ngone    = $values[6];
+    my $Nsing    = $values[7];
+    my $Ndups    = $values[8];
+    my $Nsbck    = $values[9];
+    my $Nxbck    = $values[10];
+    my $Ncul1    = $values[11];
+    my $Ncul2    = $values[12];
+    my $Ncul3    = $values[13];
+    my $Nfail    = $values[14];
+    
+    my $Ndone = $Ngood + $Ndead + $Ngone + $Nfail;
+    my $Nwait = $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3;
+  
+    if ($Ndone + $Nwait != $Nchip) { die "inconsistent result from rawfix database for $dateobs\n"; }
+
+    # check for completion here first (otherwise, advance fixchip.done below)
+    if (($state eq "fixchip.done") and ($iter > 0) and ($Nwait == 0)) { 	  &update_night_state ($dateobs, "DONE");              return "DONE"; }
+    if ($state eq "new")                          		      { 	  &update_night_state ($dateobs, "ckchip.new", 0);     return "NEW"; }
+    if ($state eq "ckchip.done")                   		      {           &update_night_state ($dateobs, "fixchip.new");       return "NEW"; }
+    if ($state eq "fixchip.done")                  		      { 	  &update_night_state ($dateobs, "md5sum.new");        return "NEW"; }
+    if ($state eq "md5sum.done")                   		      { 	  &update_night_state ($dateobs, "md5chk.new");        return "NEW"; }
+    if ($state eq "md5chk.done")                   		      { $iter ++; &update_night_state ($dateobs, "ckchip.new", $iter); return "NEW"; }
+
+    return "NONE";
+}
+
+sub update_night_state {
+    my $dateobs = $_[0];
+    my $newstate = $_[1];
+    my $newiter = $_[2];
+
+    # update the stats in the database
+    my $sql_update_state;
+    if (defined $newiter) { 
+	$sql_update_state = "UPDATE nights set state = '$newstate', iter = $newiter WHERE dateobs = $dateobs";
+	if ($DEBUG >= 2) { print "set $dateobs to $newstate, iter = $newiter\n"; }
+    } else {
+	$sql_update_state = "UPDATE nights set state = '$newstate' WHERE dateobs = $dateobs";
+	if ($DEBUG >= 2) { print "set $dateobs to $newstate\n"; }
+    }
+    
+    # print "SQL: $sql_update_state\n";
+    # return;
+
+    my $query = $raw_dbh->prepare($sql_update_state);
+    $query->execute () or die "invalid response from RAW database\n";
+    return;
+}
+
+sub parse_cmdopts {
+
+    my ($mode, $topdir, $dateobs, $fulldate, $dateword, $Nmax);
+    GetOptions( 'mode=s' => \$mode, 'topdir=s' => \$topdir, 'dateobs=s' => \$dateobs, 'Nmax=i' => \$Nmax) || pod2usage(2);
+    
+    pod2usage( -msg => "Specify the mode with --mode (all or night)", -exitval => 2) unless defined $mode;
+    pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir;
+    if ($mode eq "night") {
+	pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs;
+
+	# check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?)
+
+	my ($date_year, $date_month, $date_day) = split "/",$dateobs;
+	pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day;
+	pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030);
+	pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12);
+	pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31);
+	
+	$fulldate = "$date_year/$date_month/$date_day";
+	$dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day);
+    }
+    if ($mode eq "all") {
+	pod2usage( -msg => "Specify the maximum number of nights to advance to new with --Nmax", -exitval => 2) unless defined $Nmax;
+	# check that Nmax is an integer
+	unless (is_integer ($Nmax)) { die "Nmax must be a non-negative integer\n"; }
+	unless ($Nmax >= 0) { die "Nmax must be a non-negative integer\n"; }
+    }
+    if (($mode ne "all") and ($mode ne "night")) { die "invalid mode $mode\n"; }
+
+    return ($mode, $topdir, $fulldate, $dateword, $Nmax);
+}
+
+# parse the nebulous.config file:
+sub parse_db_config {
+
+    my $conffile = $_[0];
+
+    my ( $rawdb_name, $rawdb_host, $rawdb_port, $rawdb_pass, $rawdb_user );
+
+    if (-e $conffile) {
+	open(IN,$conffile);
+	while(<IN>) {
+	    chomp;
+	    if ($_ =~ /^\s*#/) { next; } # skip lines with leading #
+	    if ($_ =~ /RAW_NAME /)  { $rawdb_name = (split /\s+/,$_)[2]; }
+	    if ($_ =~ /RAW_HOST /)  { $rawdb_host = (split /\s+/,$_)[2]; }
+	    if ($_ =~ /RAW_USER /)  { $rawdb_user = (split /\s+/,$_)[2]; }
+	    if ($_ =~ /RAW_PASS /)  { $rawdb_pass = (split /\s+/,$_)[2]; }
+	    if ($_ =~ /RAW_PORT /)  { $rawdb_port = (split /\s+/,$_)[2]; }
+	}
+	close(IN);
+	if ($DEBUG >= 4) { print STDERR  "raw  dbh config: $rawdb_name $rawdb_host $rawdb_port $rawdb_user $rawdb_pass\n"; }
+    }
+    pod2usage( -msg => "Cannot configure raw database (dbname)", -exitval => 2) unless defined $rawdb_name;
+    pod2usage( -msg => "Cannot configure raw database (dbhost)", -exitval => 2) unless defined $rawdb_host;
+    pod2usage( -msg => "Cannot configure raw database (dbuser)", -exitval => 2) unless defined $rawdb_user;
+    pod2usage( -msg => "Cannot configure raw database (dbport)", -exitval => 2) unless defined $rawdb_port;
+    pod2usage( -msg => "Cannot configure raw database (dbpass)", -exitval => 2) unless defined $rawdb_pass;
+
+    my $rawdb_dsn = "DBI:mysql:database=$rawdb_name:host=$rawdb_host";
+    if ($rawdb_port ne "NONE") { $rawdb_dsn .= ":port=$rawdb_port"; }
+    my $raw_dbh = DBI->connect($rawdb_dsn, $rawdb_user, $rawdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n";
+
+    return ($raw_dbh);
+}
+
+sub vsystem {
+    print STDERR "@_\n";
+    # return 0;
+    my $status = system ("@_");
+    return $status;
+}
+
+sub is_integer {
+    defined $_[0] && $_[0] =~ /^[+-]?\d+$/;
+}
+
Index: /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh	(revision 42445)
+++ /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh	(revision 42446)
@@ -28,5 +28,5 @@
   endif
 
-  mysql $DBOPTS -e "select dateobs, state from nights"
+  mysql $DBOPTS -e "select dateobs, iter, state from nights"
   exit 0
 endif
@@ -49,5 +49,5 @@
 
   set mynight = $2
-  mysql $DBOPTS -e "select * from nights where (dateobs = '$mynight')"
+  mysql -E $DBOPTS -e "select * from nights where (dateobs = '$mynight')"
   exit 0
 endif
Index: /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl	(revision 42445)
+++ /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl	(revision 42446)
@@ -78,7 +78,7 @@
 
 	# determine the status of the instances
-	my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum);
-
-	&check_instance_state ($outfh, $ext_id, $data_st, $md5_sum, $Nmain, $Nback, $Ndead);
+	my ($Nmain, $Nback, $Nlost) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum);
+
+	&check_instance_state ($outfh, $ext_id, $data_st, $md5_sum, $Nmain, $Nback, $Nlost);
     }
 
@@ -239,5 +239,5 @@
     my $Nmain = 0;
     my $Nback = 0;
-    my $Ndead = 0;
+    my $Nlost = 0;
 
     # xattr = 0 means instance @ ITC  (main)
@@ -260,5 +260,5 @@
 	my $xattr    = $values[3];
 	
-	if ($xattr  > 1) { $Ndead ++; next; } # do not save instances on dead partitions
+	if ($xattr  > 1) { $Nlost ++; next; } # do not save instances on dead partitions
 	if ($xattr == 0) { $Nmain ++; }
 	if ($xattr == 1) { $Nback ++; }
@@ -277,5 +277,5 @@
     }
 
-    return ($Nmain, $Nback, $Ndead);
+    return ($Nmain, $Nback, $Nlost);
 }
 
@@ -288,5 +288,5 @@
     my $Nmain = $_[4];
     my $Nback = $_[5];
-    my $Ndead = $_[6];
+    my $Nlost = $_[6];
 
     my $outinfo = "$extID $data_st $md5_sum";
@@ -333,5 +333,5 @@
 
     # DEAD : no more valid instances
-    if (($Nmain == 0) && ($Nback == 0) && $Ndead) {
+    if (($Nmain == 0) && ($Nback == 0) && $Nlost) {
 	print $outfh "DEAD: $outinfo\n";
 	$Ndead ++;
@@ -339,5 +339,5 @@
     }
     # GONE : no more valid instances
-    if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) {
+    if (($Nmain == 0) && ($Nback == 0) && ($Nlost == 0)) {
 	print $outfh "GONE: $outinfo\n";
 	$Ngone ++;
Index: /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl	(revision 42445)
+++ /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl	(revision 42446)
@@ -40,4 +40,5 @@
 my $Ncul2 = 0;
 my $Ncul3 = 0;
+my $Nfail = 0;
 
 # load the lists of good and fail chips
@@ -146,5 +147,10 @@
 		close (OUTFILE);
 	    }
-	    if ($goodMD5) { $Ndups++; } else { $Nfail ++; }
+	    ## if we know the MD5 is bad, add to Nfail, otherwise add to Ndups
+	    if ($haveMD5 && !$goodMD5) {
+		$Nfail ++; 
+	    } else { 
+		$Ndups++; 
+	    } 
 	    next; 
 	}
@@ -162,6 +168,11 @@
 		print OUTFILE "$newfile $datast $md5sum\n";
 		close (OUTFILE);
-	    }
-	    if ($goodMD5) { $Nxbck++; } else { $Nfail ++; }
+	    } 
+	    ## if we know the MD5 is bad, add to Nfail, otherwise add to Nxbck:
+	    if ($haveMD5 && !$goodMD5) {
+		$Nfail ++; 
+	    } else { 
+		$Nxbck++; 
+	    } 
 	    next; 
 	}
@@ -193,10 +204,12 @@
 	    }
 
-	    # these are done in stage 3:
-	    if ($goodMD5) { 
+	    ## if we know the MD5 is bad, add to Nfail, otherwise add to NculX
+	    if ($haveMD5 && !$goodMD5) {
+		$Nfail ++; 
+	    } else { 
 		if ($state eq "CUL1:") { $Ncul1++; next; }
 		if ($state eq "CUL2:") { $Ncul2++; next; }
 		if ($state eq "CUL3:") { $Ncul3++; next; }
-	    } else { $Nfail ++; }
+	    } 
 	}
     }
Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.advance.pt
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/rawfix.advance.pt	(revision 42445)
+++ /trunk/tools/eam/rawfix.20230221/src/rawfix.advance.pt	(revision 42446)
@@ -17,5 +17,5 @@
   task.exec
     # 
-    command advance.rawfix.sh all $MAX_ACTIVE_NIGHTS
+    command advance.rawfix.pl --mode all --topdir $mypath --Nmax $MAX_ACTIVE_NIGHTS
   end
 
Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.pt
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/rawfix.pt	(revision 42445)
+++ /trunk/tools/eam/rawfix.20230221/src/rawfix.pt	(revision 42446)
@@ -23,5 +23,5 @@
 # order is this file is not determinative, only the
 # change of state from STAGE1.new -> STAGE1.done -> STAGE2.new
-# input rawfix.advance.pt
+input rawfix.advance.pt
 input rawfix.ckchip.pt
 input rawfix.fixchip.pt
Index: /trunk/tools/eam/rawfix.20230221/src/update.rawfix.sh
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/update.rawfix.sh	(revision 42445)
+++ /trunk/tools/eam/rawfix.20230221/src/update.rawfix.sh	(revision 42446)
@@ -2,5 +2,5 @@
 
 if ($#argv < 1) then
-  echo "USAGE: update.rawfix.sh (night) (state) [quality]"
+  echo "USAGE: update.rawfix.sh (night) (state) [iter]"
   echo "   EG: update.rawfix.sh 20100501 new"
   exit 2
@@ -9,4 +9,9 @@
 set mynight = $1
 set state   = $2
+
+set iter = ""
+if ($#argv == 3) then
+  set iter = ", iter = $3"
+endif
 
 # make the rawfix database to track rawfix operations
@@ -18,4 +23,4 @@
 set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME"
 
-mysql $DBOPTS -e "update nights set state = '$state' where (dateobs = '$mynight')"
+mysql $DBOPTS -e "update nights set state = '$state' $iter where (dateobs = '$mynight')"
 exit 0