Index: /tags/ipp-ops-20220906/ippScripts/scripts/nightly_science.pl
===================================================================
--- /tags/ipp-ops-20220906/ippScripts/scripts/nightly_science.pl	(revision 42577)
+++ /tags/ipp-ops-20220906/ippScripts/scripts/nightly_science.pl	(revision 42578)
@@ -1564,6 +1564,24 @@
         my $warps = $db->selectall_arrayref( $input_sth );
 
+        #consider which exposures should be used to be used to make warps (bad states and too high FWHM are not be allowed)
         # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
         my %comment_hash = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            my $this_quality = ${ $this_warp }[5];
+            my $this_state   = ${ $this_warp }[4];
+            my $this_fwhm   = ${ $this_warp }[6];
+	    
+            if ( ($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) {
+                print STDERR "diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n";
+            }
+            else {
+                $comment_hash{$this_comment} = $this_exp_id;
+            }
+        }
+
+        # Exclude any warps that are not stored in the comment_hash (overrides) and check their status.
+        my @keep_warps = ();
         my $Nbad = 0;
         foreach my $this_warp (@{ $warps }) {
@@ -1571,27 +1589,27 @@
             my $this_exp_id  = ${ $this_warp }[0];
             my $this_quality = ${ $this_warp }[5];
-            my $this_state   = ${ $this_warp }[4];
             my $this_warp_id  = ${ $this_warp }[1];
-            my $this_fwhm   = ${ $this_warp }[6];
             my $chip_state   = ${ $this_warp }[7];
             my $cam_state   = ${ $this_warp }[8];
-	    
-	    #find exposures for this object that have not been fully processed
-            if (($this_warp_id eq 'NULL') && (($this_quality == 0) || ($this_quality eq 'NULL')) ) {
-                print STDERR "diff_queue: exposure with exp_id $this_exp_id for $this_object has warp_id $this_warp_id and quality $this_quality and is therefore not fully processed\n";
-            	$Nbad += 1;
-            }
-            if (($chip_state ne 'full') || ($cam_state ne 'full') ) {
-                print STDERR "diff_queue: exposure with exp_id $this_exp_id for $this_object has chip state $chip_state and cam state $cam_state and is therefore not fully processed\n";
-            	$Nbad += 1;
-            }
-
-            if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) {
-                print STDERR "diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n";
+            if ((exists($comment_hash{$this_comment}))&&
+                ($comment_hash{$this_comment} == $this_exp_id)) {
+                push @keep_warps, $this_warp;
+
+	        #do not continue if you encounter exposures that finished cam stage but have not yet continued to warp (i.e. stuck in between stages)
+                if (($this_warp_id eq 'NULL') && (($this_quality == 0) || ($this_quality eq 'NULL')) ) {
+                    print STDERR "diff_queue: exposure with exp_id $this_exp_id for $this_object has warp_id $this_warp_id and quality $this_quality and is therefore not fully processed\n";
+            	    $Nbad += 1;
+                }
+	        #do not continue if you encounter exposures that have not been fully processed or have bad quality
+                if (($chip_state ne 'full') || ($cam_state ne 'full') || ($this_quality != 0) ) {
+                    print STDERR "diff_queue: exposure with exp_id $this_exp_id for $this_object has chip state $chip_state and cam state $cam_state and quality $this_quality and is therefore not fully processed\n";
+            	    $Nbad += 1;
+                }
             }
             else {
-                $comment_hash{$this_comment} = $this_exp_id;
-            }
-        }
+                print STDERR "diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n";
+            }
+        }
+        @{ $warps } = @keep_warps;
 
         #kick object out of diff consideration if it has exposures not fully processed
@@ -1600,19 +1618,4 @@
             next;
         }
-
-        # Exclude any warps that are not stored in the comment_hash.
-        my @keep_warps = ();
-        foreach my $this_warp (@{ $warps }) {
-            my $this_comment = ${ $this_warp }[3];
-            my $this_exp_id  = ${ $this_warp }[0];
-            if ((exists($comment_hash{$this_comment}))&&
-                ($comment_hash{$this_comment} == $this_exp_id)) {
-                push @keep_warps, $this_warp;
-            }
-            else {
-                print STDERR "diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n";
-            }
-        }
-        @{ $warps } = @keep_warps;
 
         # Exclude the last entry if we do not have an even number of warps.
@@ -1779,4 +1782,29 @@
             my $warpsBQ = $db->selectall_arrayref( $input_sth );;
 
+            #consider which exposures should be used to be used to make warps (bad states and too high FWHM are not be allowed)
+            # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
+            my %comment_hash = ();
+            my %comment_hash_good = ();
+            foreach my $this_warp (@{ $warps }) {
+                my $this_comment = ${ $this_warp }[3];
+                my $this_exp_id  = ${ $this_warp }[0];
+                my $this_quality = ${ $this_warp }[5];
+                my $this_state   = ${ $this_warp }[4];
+                my $this_fwhm   = ${ $this_warp }[6];
+	    
+                if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) {
+                    print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n";
+                }
+                else {
+                    $comment_hash_good{$this_comment} = $this_exp_id;
+                }
+		
+                #also save the entries which are ok, but not best quality
+                if (($this_quality == 0) && ($this_state ne 'drop') ) {
+                    $comment_hash{$this_comment} = $this_exp_id;
+                }
+		
+            }
+
             # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
             my %comment_hash = ();
@@ -1788,18 +1816,5 @@
                 my $this_quality = ${ $this_warp }[5];
                 my $this_state   = ${ $this_warp }[4];
-                my $this_warp_id  = ${ $this_warp }[1];
                 my $this_fwhm   = ${ $this_warp }[6];
-                my $chip_state   = ${ $this_warp }[7];
-                my $cam_state   = ${ $this_warp }[8];
-
-	        #find exposures for this object that have not been fully processed
-             	if (($this_warp_id eq 'NULL') && (($this_quality == 0) || ($this_quality eq 'NULL')) ) {
-                    print STDERR "desp_diff_singles: exposure with exp_id $this_exp_id for $this_object has warp_id $this_warp_id and quality $this_quality and is therefore not fully processed\n";
-             	    $Nbad += 1;
-             	}
-             	if (($chip_state ne 'full') || ($cam_state ne 'full') ) {
-                    print STDERR "desp_diff_singles: exposure with exp_id $this_exp_id for $this_object has chip state $chip_state and cam state $cam_state and is therefore not fully processed\n";
-             	    $Nbad += 1;
-             	}
 
                 if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) {
@@ -1816,4 +1831,38 @@
             }
 
+            # Exclude any warps that are not stored in the comment_hash (overrides) and check their status.
+            my @keep_warps = ();
+            my $Nbad = 0;
+            foreach my $this_warp (@{ $warps }) {
+                my $this_comment = ${ $this_warp }[3];
+                my $this_exp_id  = ${ $this_warp }[0];
+                my $this_quality = ${ $this_warp }[5];
+                my $this_warp_id  = ${ $this_warp }[1];
+                my $chip_state   = ${ $this_warp }[7];
+                my $cam_state   = ${ $this_warp }[8];
+                if ((exists($comment_hash_good{$this_comment}))&&
+                    ($comment_hash_good{$this_comment} == $this_exp_id)) {
+                    push @keep_warps, $this_warp;
+                    my $this_date  = ${ $this_warp }[2];
+                    my $chunk_name  = ${ $this_warp }[6];
+
+	            #do not continue if you encounter exposures that finished cam stage but have not yet continued to warp (i.e. stuck in between stages)
+                    if (($this_warp_id eq 'NULL') && (($this_quality == 0) || ($this_quality eq 'NULL')) ) {
+                        print STDERR "desp_diff_singles: exposure with exp_id $this_exp_id for $this_object has warp_id $this_warp_id and quality $this_quality and is therefore not fully processed\n";
+            	        $Nbad += 1;
+                    }
+	            #do not continue if you encounter exposures that have not been fully processed or have bad quality
+                    if (($chip_state ne 'full') || ($cam_state ne 'full') || ($this_quality != 0) ) {
+                        print STDERR "desp_diff_singles: exposure with exp_id $this_exp_id for $this_object has chip state $chip_state and cam state $cam_state and quality $this_quality and is therefore not fully processed\n";
+            	        $Nbad += 1;
+                    }
+                }
+                else {
+                    print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n";
+                }
+            }
+            @{ $warps } = @keep_warps;
+            my $nwarps = ($#{ $warps } + 1);
+
             #kick object out of diff consideration if it has exposures not fully processed
             if ($Nbad > 0) {
@@ -1821,22 +1870,4 @@
                 next;
             }
-
-            # Exclude any warps that are not stored in the comment_hash_good.
-            my @keep_warps = ();
-            foreach my $this_warp (@{ $warps }) {
-                my $this_comment = ${ $this_warp }[3];
-                my $this_exp_id  = ${ $this_warp }[0];
-                if ((exists($comment_hash_good{$this_comment}))&&
-                    ($comment_hash_good{$this_comment} == $this_exp_id)) {
-                    push @keep_warps, $this_warp;
-                    $this_date  = ${ $this_warp }[2];
-                    $chunk_name  = ${ $this_warp }[6];
-                }
-                else {
-                    print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n";
-                }
-            }
-            @{ $warps } = @keep_warps;
-            my $nwarps = ($#{ $warps } + 1);
            
             #find the time of the most recent exposure in that chunk
@@ -2173,4 +2204,11 @@
             my $warpsBQ = $db->selectall_arrayref( $input_sth );;
 
+
+
+
+
+
+
+
             # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
             my %comment_hash = ();
@@ -2182,18 +2220,5 @@
                 my $this_quality = ${ $this_warp }[5];
                 my $this_state   = ${ $this_warp }[4];
-                my $this_warp_id  = ${ $this_warp }[1];
-                my $this_fwhm    = ${ $this_warp }[6];
-                my $chip_state   = ${ $this_warp }[7];
-                my $cam_state   = ${ $this_warp }[8];
-
-	     	#find exposures for this object that have not been fully processed
-             	if (($this_warp_id eq 'NULL') && (($this_quality == 0) || ($this_quality eq 'NULL')) ) {
-                    print STDERR "desp_diff_queue: exposure with exp_id $this_exp_id for $this_object has warp_id $this_warp_id and quality $this_quality and is therefore not fully processed\n";
-             	    $Nbad += 1;
-             	}
-                if (($chip_state ne 'full') || ($cam_state ne 'full') ) {
-                    print STDERR "desp_diff_queue: exposure with exp_id $this_exp_id for $this_object has chip state $chip_state and cam state $cam_state and is therefore not fully processed\n";
-            	    $Nbad += 1;
-                }
+                my $this_fwhm   = ${ $this_warp }[6];
 
                 if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) {
@@ -2210,15 +2235,14 @@
             }
 
-            #kick object out of diff consideration if it has exposures not fully processed
-            if ($Nbad > 0) {
-        	print STDERR "desp_diff_queue: excluding $this_object from making diffs due to not being fully processed to warp stage\n";
-        	next;
-            }
-
-            # Exclude any warps that are not stored in the comment_hash_good.
+            # Exclude any warps that are not stored in the comment_hash (overrides) and check their status.
             my @keep_warps = ();
+            my $Nbad = 0;
             foreach my $this_warp (@{ $warps }) {
                 my $this_comment = ${ $this_warp }[3];
                 my $this_exp_id  = ${ $this_warp }[0];
+                my $this_quality = ${ $this_warp }[5];
+                my $this_warp_id  = ${ $this_warp }[1];
+                my $chip_state   = ${ $this_warp }[7];
+                my $cam_state   = ${ $this_warp }[8];
                 if ((exists($comment_hash_good{$this_comment}))&&
                     ($comment_hash_good{$this_comment} == $this_exp_id)) {
@@ -2226,11 +2250,28 @@
                     my $this_date  = ${ $this_warp }[2];
                     my $chunk_name  = ${ $this_warp }[6];
+
+	            #do not continue if you encounter exposures that finished cam stage but have not yet continued to warp (i.e. stuck in between stages)
+                    if (($this_warp_id eq 'NULL') && (($this_quality == 0) || ($this_quality eq 'NULL')) ) {
+                        print STDERR "desp_diff_queue: exposure with exp_id $this_exp_id for $this_object has warp_id $this_warp_id and quality $this_quality and is therefore not fully processed\n";
+            	        $Nbad += 1;
+                    }
+	            #do not continue if you encounter exposures that have not been fully processed or have bad quality
+                    if (($chip_state ne 'full') || ($cam_state ne 'full') || ($this_quality != 0) ) {
+                        print STDERR "desp_diff_queue: exposure with exp_id $this_exp_id for $this_object has chip state $chip_state and cam state $cam_state and quality $this_quality and is therefore not fully processed\n";
+            	        $Nbad += 1;
+                    }
                 }
                 else {
-                    print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected $this_comment\n";
+                    print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n";
                 }
             }
             @{ $warps } = @keep_warps;
             my $nwarps = ($#{ $warps } + 1);
+
+            #kick object out of diff consideration if it has exposures not fully processed
+            if ($Nbad > 0) {
+                print STDERR "desp_diff_queue: excluding $this_object from making diffs due to not being fully processed to warp stage\n";
+                next;
+            }
 
             ################################