Changeset 27838 for branches/tap_branches/tools/gpc1_find_images.pl

Timestamp:

May 3, 2010, 8:41:49 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/tap_branches

Files:

• . (modified) (1 prop)
• tools/gpc1_find_images.pl (modified) (12 diffs)

branches/tap_branches/tools/gpc1_find_images.pl

@@ -3 +3 @@
 use DBI;
 use Getopt::Std;
-getopts('hFARCPp:BMDWvx:n:c:w:O:S:',\%opt);
+getopts('hEFARCPp:BLMDWIJVYTvx:n:c:w:s:d:e:O:S:N:',\%opt);
 #getopt('xncwOS:hARCW',\%opt);
 unless (exists($opt{O})) {
@@ -34 +34 @@
 }
 
+# abcdefghijklmnopqrstuvwxyz
+#used:
+# ABCDEF     LMNOP RST  W Y
+#   c    h     n    s  vwx  
 if (exists($opt{h})) {
     print "USAGE find_images.pl ... \n";
     print "          -h              This help\n";
-    print "          -R              Return raw data.\n";
-    print "          -C              Return chip stage data.\n";
-    print "          -W              Return warp stage data.\n";
+    print "          -R              Return Raw data.\n";
+    print "          -C              Return Chip stage data.\n";
+    print "          -E              Return camEra stage data.\n";
+    print "          -W              Return Warp stage data.\n";
+    print "          -T              Get the sTacks for a given input.\n";
+    print "          -J              Get the dIff stage data (warps only for now).\n";
     print "          -A              Return not just fits images.\n";
     print "          -F              Convert to real filenames.\n";
@@ -45 +52 @@
     print "          -B              Return burntool table.\n";
     print "          -M              Include masks in results.\n";
+    print "          -L              Return the log files from this stage.\n";
     print "          -D              Return only pre-destreaked images.\n";
+    print "          -Y              Yes, I want the raw, even if it's not been processed.\n";
+    print "          -V              Return the cmf or smf files.\n";
     print "          -v              Be verbose on output.\n";
     print " \n";
     print "          -x <exp_id>     Specify a exp_id.\n";
     print "          -c <chip_id>    ...\n";
+    print "          -e <cam_id>     ...\n";
     print "          -w <warp_id>    ...\n";
+    print "          -s <stack_id>   ...\n";
+    print "          -d <diff_id>    ...\n";
     print "          -n <exp_name>   ...\n";
     print " \n";
@@ -63 +76 @@
 $dbname = 'gpc1';
 $dbserver = 'ippdb01';
-$dbuser = 'ipp';
-$dbpass = 'ipp';
+$dbuser = 'ippuser';
+$dbpass = 'ippuser';
 $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" .
                    "mysql_socket=" . DB_SOCKET(),
@@ -71 +84 @@
                    ) or die "Unable to connect to database $DBI::errstr\n";
 
-$sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING (exp_id)";
+if ($opt{Y}) {
+    $sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time,0,'nostack',camRun.cam_id,camRun.workdir,0,'nodiff' from warpRun RIGHT OUTER JOIN fakeRun USING(fake_id) RIGHT OUTER JOIN camRun USING(cam_id) RIGHT OUTER JOIN chipRun USING(chip_id) RIGHT OUTER JOIN rawExp USING (exp_id)";
+}
+else {
+    $sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time,0,'nostack',camRun.cam_id,camRun.workdir,0,'nodiff' from warpRun  JOIN fakeRun USING(fake_id)  JOIN camRun USING(cam_id)  JOIN chipRun USING(chip_id)  JOIN rawExp USING (exp_id)";
+}
+
 # What do we have?
 @sth_adds = ();
@@ -86 +105 @@
 if (exists($opt{w})) {
     push @sth_adds, "warpRun.warp_id = $opt{w}";
+}
+if (exists($opt{e})) {
+    push @sth_adds, "camRun.cam_id = $opt{e}";
+}
+if (exists($opt{s})||exists($opt{T})) {
+#    print "$opt{s}!\n";
+    if (exists($opt{s})&&($opt{s} != 0)) {
+        push @sth_adds, "stackRun.stack_id = $opt{s}";
+    }
+    $sth_base =~ s/0,'nostack'/stack_id,stackSumSkyfile.uri/;
+    if (exists($opt{T})) {
+        $sth_base .= ' RIGHT OUTER ';
+    }
+    $sth_base .= ' JOIN stackInputSkyfile USING (warp_id) JOIN stackSumSkyfile USING (stack_id) JOIN stackRun USING (stack_id)';
+#    print "$sth_base\n";
+}
+if (exists($opt{J})||(exists($opt{d}))) {
+    if (exists($opt{d})&&($opt{d} != 0)) {
+        push @sth_adds, "diffRun.diff_id = $opt{d}";
+    }
+    $sth_base =~ s/0,'nodiff'/diffRun.diff_id,diffSkyfile.path_base/;
+    if (exists($opt{J})) {
+#       $sth_base .= ' RIGHT OUTER ';
+    }
+    $sth_base .= ' JOIN diffInputSkyfile ON (warpRun.warp_id = diffInputSkyfile.warp1 OR warpRun.warp_id = diffInputSkyfile.warp2) JOIN diffRun USING (diff_id) JOIN diffSkyfile USING (diff_id)';
 }
 if ($#sth_adds != -1) {
@@ -97 +141 @@
 $dr = $db->selectall_arrayref( $sth );
 
+
+%uniq = ();
+$found = 0;
 foreach $rr (@{ $dr }) {
     ($warp_id, $warp_workdir, $chip_id, $chip_workdir, 
-     $exp_name, $exp_id, $exp_workdir, $exp_dateobs, $exp_pontime) = @{ $rr };
-#    print (join "\t", @{ $rr }) . "\n";
-    $exp_pontime = 0; # To stop warnings, but I want this around.
+     $exp_name, $exp_id, $exp_workdir, $exp_dateobs, $exp_pontime,
+     $stack_id, $stack_sumfile_uri,
+     $cam_id, $cam_workdir,
+     $diff_id, $diff_workdir
+    ) = @{ $rr };
+    if ($stack_id == 0) {
+        $stack_sumfile_uri = '';
+    }
+#    print (join "\t", @{ $rr }) . "\n" . "\n";
+#    $exp_pontime = 0; # To stop warnings, but I want this around.
+    if (exists($opt{I})) {
+        $stack_id = 0; # Ditto.
+        $cam_id = 0;
+        $diff_id = 0;
+    }
+    if (exists($opt{N})) {
+        if ($found > $opt{N}) {
+            last;
+        }
+    }
     if (exists($opt{P})) {
         $sth_previous = "SELECT exp_name from rawExp where dateobs < '$exp_dateobs' order by dateobs desc limit 1";
@@ -112 +176 @@
         if (exists($opt{B})) { $previous_cmd .= ' -B '; }
         if (exists($opt{v})) { $previous_cmd .= ' -v '; }
+        if (exists($opt{F})) { $previous_cmd .= ' -F '; }
         if ($opt{A}) { $previous_cmd .= ' -A '; }
         if ($opt{M}) { $previous_cmd .= ' -M '; }
@@ -127 +192 @@
 #       exit(0);
     }
-    
-
-
+
+    %mapping = ();
+    unless (exists($opt{Y})) {
+        if ((exists($opt{C})&&(exists($opt{S})))||
+            (exists($opt{W})&&(exists($opt{O})))||
+            exists($opt{I})) {
+            $sth_mapping = "SELECT skycell_id,class_id from warpSkyCellMap WHERE warp_id = $warp_id";
+            $mr = $db->selectall_arrayref( $sth_mapping );
+            foreach $mrr (@{ $mr }) {
+                my ($skycell,$class) = @{ $mrr };
+                my $raw_class = $class;
+                $raw_class =~ s/xy/ota/i;
+                push @{ $mapping{$skycell} }, $class;
+                push @{ $mapping{$skycell} }, $raw_class;
+                push @{ $mapping{$class} }, $skycell;
+                push @{ $mapping{$raw_class} }, $skycell;
+#           print STDERR "$raw_class $class $skycell\n";
+            }
+        }
+    }
+    
+    if (exists($opt{I})) {
+        if ($warp_id eq '') {
+            $warp_id = 'NULL';
+        }
+        if ($stack_id eq '') {
+            $stack_id = 'NULL';
+        }
+        $line = "$exp_name $exp_id $exp_dateobs $exp_pontime $chip_id $warp_id\t";
+        if (exists($opt{O})) {
+            foreach $sky (@{ $mapping{$opt{O}} }) {
+                $line .= "$sky\t";
+            }
+        }
+        if (exists($opt{S})) {
+            foreach $sky (@{ $mapping{$opt{S}} }) {
+                $line .= "$sky\t";
+            }
+        }
+        $line .= "\n";
+        unless (exists($uniq{$line})) {
+            print $line;
+            $uniq{$line} = 1;
+        }
+        next;
+    }
     @scan = ();
     if (exists($opt{R})||exists($opt{B})) {
+#       print STDERR "$exp_workdir $exp_name\n";
         open(NLS,"neb-ls $exp_workdir/${exp_name}/ |") || die "can't neb-ls";
         while (<NLS>) {
             chomp;
-            if (($_ =~ /${opt{O}}/)&&((($opt{A})) || ($_ =~ /fits/))) {
-                if (exists($opt{B})) {
-                    $_ =~ s/.fits$/.burn.tbl/;
-                }
+            if (is_valid_raw($_)) {
                 push @scan, $_;
+                $found++;
             }
         }
@@ -148 +255 @@
         while (<NLS>) {
             chomp;
-            if (($_ =~ /${exp_id}.ch/)&&($_ =~ /$chip_id/)&&($_ =~ /$opt{O}/)&&
-                ((($opt{A})) || (($_ =~ /fits/) &&
-                                 ($opt{M} || ($_ !~ /mk.fits/)) &&
-                                 ($_ !~ /wt.fits/)))) {
-                if ($opt{D} && ($_ !~ /SR/)) {
-                    next;
-                }
+            if (is_valid_chip($_)) {
                 push @scan, $_;
-
+                $found++;
+            }
+        }
+        close(NLS);
+    }
+    if (exists($opt{E})) {
+        open(NLS,"neb-ls $cam_workdir/${exp_name}.${exp_id}/ |") || die "can't neb-ls";
+        while (<NLS>) {
+            chomp;
+            if (is_valid_cam($_)) {
+                push @scan, $_;
+                $found++;
             }
         }
@@ -165 +277 @@
         while (<NLS>) {
             chomp;
-#           print "$_\n";
-            if (($_ =~ /wrp.${warp_id}.$opt{S}/) && ((($opt{A})) || (($_ =~ /fits/) &&
-                                                                     ($opt{M} || ($_ !~ /mask/)) &&
-                                                                     ($_ !~ /wt/)))) {
-                if ($opt{D} && ($_ !~ /SR/)) {
-                    next;
-                }
+            if (is_valid_warp($_)) {
                 push @scan, $_;
-            }
-        }
-    }
-    %uniq = ();
+                $found++;
+            }
+        }
+    }
+    if (exists($opt{J})) {
+        @diff_tmp = split /\//, $diff_workdir;
+        $diff_prefix = pop(@diff_tmp);
+        $diff_true_dir = join '/',@diff_tmp;
+
+        open(NLS,"neb-ls $diff_true_dir |") || die "can't neb-ls";
+        while (<NLS>) {
+            chomp;
+            if (is_valid_diff($_)) {
+                push @scan, $_;
+                $found++;
+            }
+        }
+    }
+    if (exists($opt{s})||exists($opt{T})) {
+        if (defined($stack_sumfile_uri)) {
+            push @scan, $stack_sumfile_uri;
+            $found++;
+        }
+    }
+
     foreach $f (@scan) {
         unless (exists($uniq{$f})) {
@@ -190 +317 @@
     }
 }
+
+sub is_mapped {
+    my ($key,$map_string) = @_;
+    
+    unless (exists($mapping{$map_string})) {
+        return(0);
+    }
+    
+    foreach $v (@{ $mapping{$map_string} }) {
+        if ($key =~ /$v/) {
+            return(1);
+        }
+    }
+    return(0);
+}
+
+sub is_valid_raw {
+    my $key = shift;
+
+    if ($opt{A}) {
+        return(1);
+    }
+    elsif ($opt{L}) {
+        if ($key =~ /log/) {
+            return(1);
+        }
+        else {
+            return(0);
+        }
+    }
+    elsif ($opt{B}) {
+        if ($key =~ /burn.tbl/) {
+            if (($opt{O} ne '')&&($key !~ /$opt{O}/)) {
+                return(0); # Not the requested OTA
+            }
+            if (($opt{S} ne '')&& !(is_mapped($key,$opt{S}))) {
+                return(0); # Not the remapped skycell requested
+            }
+            return(1);
+        }
+        else {
+            return(0);
+        }
+    }
+    elsif ($key !~ /fits/) {
+        return(0);
+    }
+    else {
+        if (($opt{O} ne '')&&($key !~ /$opt{O}/)) {
+            return(0); # Not the requested OTA
+        }
+        if (($opt{S} ne '')&& !(is_mapped($key,$opt{S}))) {
+            return(0); # Not the remapped skycell requested
+        }
+        if ($key =~ /burn.fits/) {
+            return(0);
+        }
+        return(1);
+    }
+}
+    
+sub is_valid_chip {
+    my $key = shift;
+    
+    
+    if ($opt{D} && ($key !~ /SR/)) {
+        return(0); # Not destreaked when required
+    }
+    elsif ($key !~ /${exp_id}.ch/) {
+        return(0); # Not a chip from this exposure
+    }
+    elsif ($key !~ /${chip_id}/) {
+        return(0); # Not from this chip_id
+    }
+
+    if (($opt{O} ne '')&&($key !~ /$opt{O}/)) {
+        return(0); # Not the requested OTA
+    }
+    if (($opt{S} ne '')&& !(is_mapped($key,$opt{S}))) {
+        return(0); # Not the remapped skycell requested
+    }
+            
+    if ($opt{A}) {
+        return(1); # You asked for all, 
+    }
+    elsif ($opt{L}) {
+        if ($key =~ /log/) {
+            return(1);
+        }
+        else {
+            return(0);
+        }
+    }
+    elsif ($opt{V} && ($key =~ /cmf/)) {
+        return(1);
+    }
+    elsif ($key !~ /fits/) {
+        return(0); # Not a fits
+    }
+    elsif ($key =~ /wt.fits/) {
+        return(0); # No handler for weight right now.
+    }
+    elsif ($key =~ /mk/) {
+        if ($opt{M}) {
+            return(1); # Requested mask
+        }
+        else {
+            return(0);
+        }
+    }
+    elsif ($key =~ /ch.fits/) {
+        return(1); # Should be just the chip
+    }
+    return(0);
+
+}
+
+sub is_valid_cam {
+    my $key = shift;
+    if ($key =~ /\.cm\./) {
+        if ($opt{A}) {
+            return(1);
+        }
+        elsif ($opt{L} && ($key =~ /log/)) {
+            return(1);
+        }
+        elsif ($opt{V} && ($key =~ /smf/)) {
+            return(1);
+        }
+        elsif ($opt{M} && ($key =~ /mk.fits/)) {
+            if (($opt{O} ne '')&&($key !~ /$opt{O}/)) {
+                return(0); # Not the requested OTA
+            }
+            if (($opt{S} ne '')&& !(is_mapped($key,$opt{S}))) {
+                return(0); # Not the remapped skycell requested
+            }
+            return(1);
+        }           
+
+    }
+    return(0);
+}
+
+sub is_valid_diff {
+    my $key = shift;
+    if ($key =~ /$diff_prefix/) {
+        if ($opt{D} && ($key !~ /SR/)) {
+            return(0); # Not destreaked when required
+        }
+        if (($opt{S} ne '')&&($key !~ /$opt{S}/)) {
+            return(0); # Not the requested skycell
+        }
+        if (($opt{O} ne '')&& !(is_mapped($key,$opt{O}))) {
+            return(0); # Not the remapped OTA requested
+        }
+        
+        if ($opt{A}) {
+            return(1); # You asked for all, 
+        }
+        elsif ($opt{L}) {
+            if ($key =~ /log/) {
+                return(1);
+            }
+            else {
+                return(0);
+            }
+        }
+        elsif ($opt{V} && ($key =~ /cmf/)) {
+            return(1);
+        }
+        elsif ($key !~ /fits/) {
+            return(0); # Not a fits
+        }
+        elsif ($key =~ /refConv/) {
+            return(0);
+        }
+        elsif ($key =~ /inConv/) {
+            return(0);
+        }
+        elsif ($key =~ /inv/) {
+            return(0);
+        }
+        elsif ($key =~ /wt.fits/) {
+            return(0); # No handler for weight right now.
+        }
+
+        elsif ($key =~ /mask/) {
+            if ($opt{M}) {
+                return(1); # Requested mask
+            }
+            else {
+                return(0);
+            }
+        }
+        else {
+            return(1); # Should be just the warp.
+        }
+    }
+    
+    return(0);
+}
+sub is_valid_warp {
+    my $key = shift;
+    
+    if ($opt{D} && ($key !~ /SR/)) {
+        return(0); # Not destreaked when required
+    }
+    elsif ($key !~ /wrp.${warp_id}/) {
+        return(0); # Not a warp from this warp_id
+    }
+    if (($opt{S} ne '')&&($key !~ /$opt{S}/)) {
+        return(0); # Not the requested skycell
+    }
+    if (($opt{O} ne '')&& !(is_mapped($key,$opt{O}))) {
+        return(0); # Not the remapped OTA requested
+    }
+            
+    if ($opt{A}) {
+        return(1); # You asked for all, 
+    }
+    elsif ($opt{L}) {
+        if ($key =~ /log/) {
+            return(1);
+        }
+        else {
+            return(0);
+        }
+    }
+    elsif ($opt{V} && ($key =~ /cmf/)) {
+        return(1);
+    }
+    elsif ($key !~ /fits/) {
+        return(0); # Not a fits
+    }
+    elsif ($key =~ /wt.fits/) {
+        return(0); # No handler for weight right now.
+    }
+    elsif ($key =~ /mask/) {
+        if ($opt{M}) {
+            return(1); # Requested mask
+        }
+        else {
+            return(0);
+        }
+    }
+    else {
+        return(1); # Should be just the warp.
+    }
+    return(0);
+}