Index: trunk/tools/find_images.pl
===================================================================
--- trunk/tools/find_images.pl	(revision 41314)
+++ trunk/tools/find_images.pl	(revision 41314)
@@ -0,0 +1,571 @@
+#! /usr/bin/perl -w
+
+use DBI;
+use Getopt::Std;
+getopts('hEFARCPp:BLMDWIJVYTvx:n:c:w:s:d:e:O:S:N:z:',\%opt);
+#getopt('xncwOS:hARCW',\%opt);
+unless (exists($opt{z})) {
+    $opt{z} = 'gpc1';
+}
+unless (exists($opt{O})) {
+    $opt{O} = '';
+}
+unless (exists($opt{S})) {
+    $opt{S} = '';
+}
+if (exists($opt{A})) {
+    $opt{A} = 1;
+}
+else {
+    $opt{A} = 0;
+}
+if (exists($opt{M})) {
+    $opt{M} = 1;
+}
+else {
+    $opt{M} = 0;
+}
+if (exists($opt{D})) {
+    $opt{D} = 1;
+}
+else {
+    $opt{D} = 0;
+}
+
+unless (exists($opt{p})) {
+    $opt{p} = 0;
+}
+
+# abcdefghijklmnopqrstuvwxyz
+#used:
+# ABCDEF     LMNOP RST  W Y
+#   c    h     n    s  vwx  
+if (exists($opt{h})) {
+    print "USAGE find_images.pl ... \n";
+    print "          -h              This help\n";
+    print "          -R              Return Raw data.\n";
+    print "          -C              Return Chip stage data.\n";
+    print "          -E              Return camEra stage data.\n";
+    print "          -W              Return Warp stage data.\n";
+    print "          -T              Get the sTacks for a given input.\n";
+    print "          -J              Get the dIff stage data (warps only for now).\n";
+    print "          -A              Return not just fits images.\n";
+    print "          -F              Convert to real filenames.\n";
+    print "          -P              Return previous image to this.\n";
+    print "          -B              Return burntool table.\n";
+    print "          -M              Include masks in results.\n";
+    print "          -L              Return the log files from this stage.\n";
+    print "          -D              Return only pre-destreaked images.\n";
+    print "          -Y              Yes, I want the raw, even if it's not been processed.\n";
+    print "          -V              Return the cmf or smf files.\n";
+    print "          -v              Be verbose on output.\n";
+    print " \n";
+    print "          -x <exp_id>     Specify a exp_id.\n";
+    print "          -c <chip_id>    ...\n";
+    print "          -e <cam_id>     ...\n";
+    print "          -w <warp_id>    ...\n";
+    print "          -s <stack_id>   ...\n";
+    print "          -d <diff_id>    ...\n";
+    print "          -n <exp_name>   ...\n";
+    print "          -z <dbname>    ...\n";
+    print " \n";
+    print "          -O <XYxy>       Return a specific chip.\n";
+    print "          -S <skycell.X>  Return a specific skycell.\n";
+    exit(0);
+}
+
+# Set up the database
+use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
+$dbname = $opt{z};
+$dbserver = 'ippdb08';
+$dbuser = 'ippuser';
+$dbpass = 'ippuser';
+$db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" .
+                   "mysql_socket=" . DB_SOCKET(),
+                   ${dbuser},${dbpass}, 
+		   { RaiseError => 1, AutoCommit => 1}
+		   ) or die "Unable to connect to database $DBI::errstr\n";
+
+if ($opt{Y}) {
+    $sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time,0,'nostack',camRun.cam_id,camRun.workdir,0,'nodiff' from warpRun RIGHT OUTER JOIN fakeRun USING(fake_id) RIGHT OUTER JOIN camRun USING(cam_id) RIGHT OUTER JOIN chipRun USING(chip_id) RIGHT OUTER JOIN rawExp USING (exp_id)";
+}
+else {
+    $sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time,0,'nostack',camRun.cam_id,camRun.workdir,0,'nodiff' from warpRun  JOIN fakeRun USING(fake_id)  JOIN camRun USING(cam_id)  JOIN chipRun USING(chip_id)  JOIN rawExp USING (exp_id)";
+}
+
+# What do we have?
+@sth_adds = ();
+
+if (exists($opt{x})) {
+    push @sth_adds, "rawExp.exp_id = $opt{x}";
+}
+if (exists($opt{n})) {
+    push @sth_adds, "rawExp.exp_name = '$opt{n}'";
+}
+if (exists($opt{c})) {
+    push @sth_adds, "chipRun.chip_id = $opt{c}";
+}
+if (exists($opt{w})) {
+    push @sth_adds, "warpRun.warp_id = $opt{w}";
+}
+if (exists($opt{e})) {
+    push @sth_adds, "camRun.cam_id = $opt{e}";
+}
+if (exists($opt{s})||exists($opt{T})) {
+#    print "$opt{s}!\n";
+    if (exists($opt{s})&&($opt{s} != 0)) {
+	push @sth_adds, "stackRun.stack_id = $opt{s}";
+    }
+    $sth_base =~ s/0,'nostack'/stack_id,stackSumSkyfile.uri/;
+    if (exists($opt{T})) {
+	$sth_base .= ' RIGHT OUTER ';
+    }
+    $sth_base .= ' JOIN stackInputSkyfile USING (warp_id) JOIN stackSumSkyfile USING (stack_id) JOIN stackRun USING (stack_id)';
+#    print "$sth_base\n";
+}
+if (exists($opt{J})||(exists($opt{d}))) {
+    if (exists($opt{d})&&($opt{d} != 0)) {
+	push @sth_adds, "diffRun.diff_id = $opt{d}";
+    }
+    $sth_base =~ s/0,'nodiff'/diffRun.diff_id,diffSkyfile.path_base/;
+    if (exists($opt{J})) {
+#	$sth_base .= ' RIGHT OUTER ';
+    }
+    $sth_base .= ' JOIN diffInputSkyfile ON (warpRun.warp_id = diffInputSkyfile.warp1 OR warpRun.warp_id = diffInputSkyfile.warp2) JOIN diffRun USING (diff_id) JOIN diffSkyfile USING (diff_id)';
+}
+if ($#sth_adds != -1) {
+    $sth_add = join ' AND ', @sth_adds;
+    $sth = $sth_base . " WHERE " . $sth_add;
+}
+else {
+    die "Probable mistake.";
+}
+#print "### $sth\n";
+$dr = $db->selectall_arrayref( $sth );
+
+
+%uniq = ();
+$found = 0;
+foreach $rr (@{ $dr }) {
+    ($warp_id, $warp_workdir, $chip_id, $chip_workdir, 
+     $exp_name, $exp_id, $exp_workdir, $exp_dateobs, $exp_pontime,
+     $stack_id, $stack_sumfile_uri,
+     $cam_id, $cam_workdir,
+     $diff_id, $diff_workdir
+    ) = @{ $rr };
+    if ($stack_id == 0) {
+	$stack_sumfile_uri = '';
+    }
+#    print (join "\t", @{ $rr }) . "\n" . "\n";
+#    $exp_pontime = 0; # To stop warnings, but I want this around.
+    if (exists($opt{I})) {
+	$stack_id = 0; # Ditto.
+	$cam_id = 0;
+	$diff_id = 0;
+    }
+    if (exists($opt{N})) {
+	if ($found > $opt{N}) {
+	    last;
+	}
+    }
+    if (exists($opt{P})) {
+	$sth_previous = "SELECT exp_name from rawExp where dateobs < '$exp_dateobs' order by dateobs desc limit 1";
+	$prev_dr = $db->selectall_arrayref( $sth_previous );
+	$prev_exp_name = ${ ${ $prev_dr }[0] }[0];
+	$previous_cmd = "$0  -n $prev_exp_name";
+	if (exists($opt{R})) { $previous_cmd .= ' -R '; }
+	if (exists($opt{C})) { $previous_cmd .= ' -C '; }
+	if (exists($opt{W})) { $previous_cmd .= ' -W '; }
+	if (exists($opt{B})) { $previous_cmd .= ' -B '; }
+	if (exists($opt{v})) { $previous_cmd .= ' -v '; }
+	if (exists($opt{F})) { $previous_cmd .= ' -F '; }
+	if ($opt{A}) { $previous_cmd .= ' -A '; }
+	if ($opt{M}) { $previous_cmd .= ' -M '; }
+	if ($opt{D}) { $previous_cmd .= ' -D '; }
+	if ($opt{O} ne '') { $previous_cmd .= " -O $opt{O} "; }
+	if ($opt{S} ne '') { $previous_cmd .= " -S $opt{S} "; }
+	if ($opt{p} > 1) {
+	    $opt{p} -= 1;
+	    $previous_cmd .= " -P -p $opt{p} ";
+	}
+	if (exists($opt{v})) {
+	    print STDERR "# $previous_cmd\n";
+	}
+	system($previous_cmd);
+#	exit(0);
+    }
+
+    %mapping = ();
+    unless (exists($opt{Y})) {
+	if ((exists($opt{C})&&(exists($opt{S})))||
+	    (exists($opt{W})&&(exists($opt{O})))||
+	    exists($opt{I})) {
+	    $sth_mapping = "SELECT skycell_id,class_id from warpSkyCellMap WHERE warp_id = $warp_id";
+	    $mr = $db->selectall_arrayref( $sth_mapping );
+	    foreach $mrr (@{ $mr }) {
+		my ($skycell,$class) = @{ $mrr };
+		my $raw_class = $class;
+		$raw_class =~ s/xy/ota/i;
+		push @{ $mapping{$skycell} }, $class;
+		push @{ $mapping{$skycell} }, $raw_class;
+		push @{ $mapping{$class} }, $skycell;
+		push @{ $mapping{$raw_class} }, $skycell;
+#	    print STDERR "$raw_class $class $skycell\n";
+	    }
+	}
+    }
+    
+    if (exists($opt{I})) {
+	if ($warp_id eq '') {
+	    $warp_id = 'NULL';
+	}
+	if ($stack_id eq '') {
+	    $stack_id = 'NULL';
+	}
+	$line = "$exp_name $exp_id $exp_dateobs $exp_pontime $chip_id $warp_id\t";
+	if (exists($opt{O})) {
+	    foreach $sky (@{ $mapping{$opt{O}} }) {
+		$line .= "$sky\t";
+	    }
+	}
+	if (exists($opt{S})) {
+	    foreach $sky (@{ $mapping{$opt{S}} }) {
+		$line .= "$sky\t";
+	    }
+	}
+	$line .= "\n";
+	unless (exists($uniq{$line})) {
+	    print $line;
+	    $uniq{$line} = 1;
+	}
+	next;
+    }
+    @scan = ();
+    if (exists($opt{R})||exists($opt{B})) {
+#	print STDERR "$exp_workdir $exp_name\n";
+	open(NLS,"neb-ls $exp_workdir/${exp_name}/ |") || die "can't neb-ls";
+	while (<NLS>) {
+	    chomp;
+	    if (is_valid_raw($_)) {
+		push @scan, $_;
+		$found++;
+	    }
+	}
+	close(NLS);
+    }
+    if (exists($opt{C})) {
+	open(NLS,"neb-ls $chip_workdir/${exp_name}.${exp_id}/ |") || die "can't neb-ls";
+	while (<NLS>) {
+	    chomp;
+	    if (is_valid_chip($_)) {
+		push @scan, $_;
+		$found++;
+	    }
+	}
+	close(NLS);
+    }
+    if (exists($opt{E})) {
+	open(NLS,"neb-ls $cam_workdir/${exp_name}.${exp_id}/ |") || die "can't neb-ls";
+	while (<NLS>) {
+	    chomp;
+	    if (is_valid_cam($_)) {
+		push @scan, $_;
+		$found++;
+	    }
+	}
+	close(NLS);
+    }
+    if (exists($opt{W})) {
+	open(NLS,"neb-ls $warp_workdir/${exp_name}.${exp_id}/ |") || die "can't neb-ls";
+	while (<NLS>) {
+	    chomp;
+	    if (is_valid_warp($_)) {
+		push @scan, $_;
+		$found++;
+	    }
+	}
+    }
+    if (exists($opt{J})) {
+	@diff_tmp = split /\//, $diff_workdir;
+	$diff_prefix = pop(@diff_tmp);
+	$diff_true_dir = join '/',@diff_tmp;
+
+	open(NLS,"neb-ls $diff_true_dir |") || die "can't neb-ls";
+	while (<NLS>) {
+	    chomp;
+	    if (is_valid_diff($_)) {
+		push @scan, $_;
+		$found++;
+	    }
+	}
+    }
+    if (exists($opt{s})||exists($opt{T})) {
+	if (defined($stack_sumfile_uri)) {
+	    push @scan, $stack_sumfile_uri;
+	    $found++;
+	}
+    }
+
+    foreach $f (@scan) {
+	unless (exists($uniq{$f})) {
+	    $uniq{$f} = 1;
+	    if (exists($opt{F})) {
+		chomp($f = `neb-locate --path $f`);
+	    }
+	    print "$f\n";
+	    if (exists($opt{v})) {
+		print STDERR "$f\n";
+	    }
+	}
+    }
+}
+
+sub is_mapped {
+    my ($key,$map_string) = @_;
+    
+    unless (exists($mapping{$map_string})) {
+	return(0);
+    }
+    
+    foreach $v (@{ $mapping{$map_string} }) {
+	if ($key =~ /$v/) {
+	    return(1);
+	}
+    }
+    return(0);
+}
+
+sub is_valid_raw {
+    my $key = shift;
+
+    if ($opt{A}) {
+	return(1);
+    }
+    elsif ($opt{L}) {
+	if ($key =~ /log/) {
+	    return(1);
+	}
+	else {
+	    return(0);
+	}
+    }
+    elsif ($opt{B}) {
+	if ($key =~ /burn.tbl/) {
+	    if (($opt{O} ne '')&&($key !~ /$opt{O}/)) {
+		return(0); # Not the requested OTA
+	    }
+	    if (($opt{S} ne '')&& !(is_mapped($key,$opt{S}))) {
+		return(0); # Not the remapped skycell requested
+	    }
+	    return(1);
+	}
+	else {
+	    return(0);
+	}
+    }
+    elsif ($key !~ /fits/) {
+	return(0);
+    }
+    else {
+	if (($opt{O} ne '')&&($key !~ /$opt{O}/)) {
+	    return(0); # Not the requested OTA
+	}
+	if (($opt{S} ne '')&& !(is_mapped($key,$opt{S}))) {
+	    return(0); # Not the remapped skycell requested
+	}
+	if ($key =~ /burn.fits/) {
+	    return(0);
+	}
+	return(1);
+    }
+}
+    
+sub is_valid_chip {
+    my $key = shift;
+    
+    
+    if ($opt{D} && ($key !~ /SR/)) {
+	return(0); # Not destreaked when required
+    }
+    elsif ($key !~ /${exp_id}.ch/) {
+	return(0); # Not a chip from this exposure
+    }
+    elsif ($key !~ /${chip_id}/) {
+	return(0); # Not from this chip_id
+    }
+
+    if (($opt{O} ne '')&&($key !~ /$opt{O}/)) {
+	return(0); # Not the requested OTA
+    }
+    if (($opt{S} ne '')&& !(is_mapped($key,$opt{S}))) {
+	return(0); # Not the remapped skycell requested
+    }
+	    
+    if ($opt{A}) {
+	return(1); # You asked for all, 
+    }
+    elsif ($opt{L}) {
+	if ($key =~ /log/) {
+	    return(1);
+	}
+	else {
+	    return(0);
+	}
+    }
+    elsif ($opt{V} && ($key =~ /cmf/)) {
+	return(1);
+    }
+    elsif ($key !~ /fits/) {
+	return(0); # Not a fits
+    }
+    elsif ($key =~ /wt.fits/) {
+	return(0); # No handler for weight right now.
+    }
+    elsif ($key =~ /mk/) {
+	if ($opt{M}) {
+	    return(1); # Requested mask
+	}
+	else {
+	    return(0);
+	}
+    }
+    elsif ($key =~ /ch.fits/) {
+	return(1); # Should be just the chip
+    }
+    return(0);
+
+}
+
+sub is_valid_cam {
+    my $key = shift;
+    if ($key =~ /\.cm\./) {
+	if ($opt{A}) {
+	    return(1);
+	}
+	elsif ($opt{L} && ($key =~ /log/)) {
+	    return(1);
+	}
+	elsif ($opt{V} && ($key =~ /smf/)) {
+	    return(1);
+	}
+	elsif ($opt{M} && ($key =~ /mk.fits/)) {
+	    if (($opt{O} ne '')&&($key !~ /$opt{O}/)) {
+		return(0); # Not the requested OTA
+	    }
+	    if (($opt{S} ne '')&& !(is_mapped($key,$opt{S}))) {
+		return(0); # Not the remapped skycell requested
+	    }
+	    return(1);
+	}	    
+
+    }
+    return(0);
+}
+
+sub is_valid_diff {
+    my $key = shift;
+    if ($key =~ /$diff_prefix/) {
+	if ($opt{D} && ($key !~ /SR/)) {
+	    return(0); # Not destreaked when required
+	}
+	if (($opt{S} ne '')&&($key !~ /$opt{S}/)) {
+	    return(0); # Not the requested skycell
+	}
+	if (($opt{O} ne '')&& !(is_mapped($key,$opt{O}))) {
+	    return(0); # Not the remapped OTA requested
+	}
+	
+	if ($opt{A}) {
+	    return(1); # You asked for all, 
+	}
+	elsif ($opt{L}) {
+	    if ($key =~ /log/) {
+		return(1);
+	    }
+	    else {
+		return(0);
+	    }
+	}
+	elsif ($opt{V} && ($key =~ /cmf/)) {
+	    return(1);
+	}
+	elsif ($key !~ /fits/) {
+	    return(0); # Not a fits
+	}
+	elsif ($key =~ /refConv/) {
+	    return(0);
+	}
+	elsif ($key =~ /inConv/) {
+	    return(0);
+	}
+	elsif ($key =~ /inv/) {
+	    return(0);
+	}
+	elsif ($key =~ /wt.fits/) {
+	    return(0); # No handler for weight right now.
+	}
+
+	elsif ($key =~ /mask/) {
+	    if ($opt{M}) {
+		return(1); # Requested mask
+	    }
+	    else {
+		return(0);
+	    }
+	}
+	else {
+	    return(1); # Should be just the warp.
+	}
+    }
+    
+    return(0);
+}
+sub is_valid_warp {
+    my $key = shift;
+    
+    if ($opt{D} && ($key !~ /SR/)) {
+	return(0); # Not destreaked when required
+    }
+    elsif ($key !~ /wrp.${warp_id}/) {
+	return(0); # Not a warp from this warp_id
+    }
+    if (($opt{S} ne '')&&($key !~ /$opt{S}/)) {
+	return(0); # Not the requested skycell
+    }
+    if (($opt{O} ne '')&& !(is_mapped($key,$opt{O}))) {
+	return(0); # Not the remapped OTA requested
+    }
+	    
+    if ($opt{A}) {
+	return(1); # You asked for all, 
+    }
+    elsif ($opt{L}) {
+	if ($key =~ /log/) {
+	    return(1);
+	}
+	else {
+	    return(0);
+	}
+    }
+    elsif ($opt{V} && ($key =~ /cmf/)) {
+	return(1);
+    }
+    elsif ($key !~ /fits/) {
+	return(0); # Not a fits
+    }
+    elsif ($key =~ /wt.fits/) {
+	return(0); # No handler for weight right now.
+    }
+    elsif ($key =~ /mask/) {
+	if ($opt{M}) {
+	    return(1); # Requested mask
+	}
+	else {
+	    return(0);
+	}
+    }
+    else {
+	return(1); # Should be just the warp.
+    }
+    return(0);
+}