Index: branches/czw_branch/20170908/ippScripts/Build.PL =================================================================== --- branches/czw_branch/20170908/ippScripts/Build.PL (revision 40128) +++ branches/czw_branch/20170908/ippScripts/Build.PL (revision 40483) @@ -115,4 +115,5 @@ scripts/automate_stacks.pl scripts/nightly_science.pl + scripts/night_report.pl scripts/lossy_compress_imfile.pl scripts/rawcheck.pl Index: branches/czw_branch/20170908/ippScripts/scripts/camera_exp.pl =================================================================== --- branches/czw_branch/20170908/ippScripts/scripts/camera_exp.pl (revision 40128) +++ branches/czw_branch/20170908/ippScripts/scripts/camera_exp.pl (revision 40483) @@ -404,5 +404,5 @@ # Construct FPA continuity corrected background images - if (($camera =~ /ISP/)||($camera =~ /HSC/)) { + if (($camera =~ /ISP/)||($camera =~ /HSC/)||($camera =~ /gpc2/i)) { print "Skipping FPA continuity corrected background images for ISP\n"; } elsif ($do_bkg) { Index: branches/czw_branch/20170908/ippScripts/scripts/chip_imfile.pl =================================================================== --- branches/czw_branch/20170908/ippScripts/scripts/chip_imfile.pl (revision 40128) +++ branches/czw_branch/20170908/ippScripts/scripts/chip_imfile.pl (revision 40483) @@ -632,4 +632,5 @@ if ($gone) { carp "WARNING: photometry sources storage object exists but all instances are permanently gone"; + $ipprc->file_rename($outputSources,$outputSources . ".gone"); $make_sources = 1; } @@ -657,4 +658,5 @@ if ($gone) { carp "WARNING: PSF storage object exists but all instances are permanently gone"; + $ipprc->file_rename($outputPsf,$outputPsf . ".gone"); $make_psf = 1; } @@ -742,4 +744,6 @@ print STDERR "$file is on $volume which is gone\n"; $numGone++; + } elsif ($answer eq 'permanently') { + $numGone++; } elsif ($answer eq 'available') { $numNotGone++; Index: branches/czw_branch/20170908/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- branches/czw_branch/20170908/ippScripts/scripts/ipp_cleanup.pl (revision 40128) +++ branches/czw_branch/20170908/ippScripts/scripts/ipp_cleanup.pl (revision 40483) @@ -27,5 +27,5 @@ # turn this on the set the state to error_state when errors occur cleaning up individual components # We may stop doing this -my $set_error_state_for_run = 1; +my $set_error_state_for_run = 0; # this gets set to 1 the first time we set the corresponding destreak run to be cleaned Index: branches/czw_branch/20170908/ippScripts/scripts/ipp_filename.pl =================================================================== --- branches/czw_branch/20170908/ippScripts/scripts/ipp_filename.pl (revision 40128) +++ branches/czw_branch/20170908/ippScripts/scripts/ipp_filename.pl (revision 40483) @@ -10,10 +10,15 @@ #print "$ENV{'PATH'}\n"; #print "$ENV{'PERL5LIB'}\n"; +#print "NEB_SERVER: $ENV{'NEB_SERVER'}
\n"; -# CZW: This is a horrible hack, but I don't want to have to debug all of ippMonitor to figure out why it's not working. +# EAM: check for missing NEB_SERVER and exit with an error unless (defined($ENV{'NEB_SERVER'})) { - $ENV{'NEB_SERVER'} = 'http://nebserver.ipp.ifa.hawaii.edu:80/nebulous'; + print "NEB_SERVER not defined in ipp_filename.pl\n"; + exit (5); } - +if ($ENV{'NEB_SERVER'} eq "") { + print "NEB_SERVER not set in ipp_filename.pl\n"; + exit (6); +} use PS::IPP::Config; @@ -52,4 +57,9 @@ my $realname = $ipprc->file_resolve( $filename, $touch ); +if (not defined $realname) { + print "nebulous file $filename not found\n"; + exit (1); +} + print "$realname\n"; Index: branches/czw_branch/20170908/ippScripts/scripts/night_report.pl =================================================================== --- branches/czw_branch/20170908/ippScripts/scripts/night_report.pl (revision 40483) +++ branches/czw_branch/20170908/ippScripts/scripts/night_report.pl (revision 40483) @@ -0,0 +1,733 @@ +#!/usr/bin/env perl + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use DBI; +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; # Socket for mysql + +# this saves the original arguments for later reporting +my @ARGS = @ARGV; + +GetOptions( + 'date=s' => \$date, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'very-verbose' => \$verbose2, # Print to stdout + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option(s): @ARGV", -exitval => 2 ) if @ARGV; + +# example if I require some options +# pod2usage( -msg => "Required options: --exp_id --chip_id", -exitval => 3) unless defined $exp_id and defined $chip_id; + +$date = `date --utc --rfc-3339=date` unless defined $date; +chomp $date; +$date=~ s/\//-/g; +$date=~ s/\./-/g; + +# database connection information: +$dbhost = 'ippdb08'; +$dbname = 'gpc1' unless defined $dbname; +$dbuser = 'ippuser'; +$dbpass = 'ippuser'; + +# dataGroupDate : formatted to match data_group values (YYYYMMDD) +$dataGroupDate = $date; +$dataGroupDate=~ s/-//g; +$dataGroupDate=~ s/\.//g; + +print "\n\n\n"; +print "--- Report for $dbname : $date ---\n\n"; + +# set up the database connection: +my $dsn = "DBI:mysql:host=$dbhost;database=$dbname"; +my $dbh = DBI->connect( $dsn, $dbuser, $dbpass ) or die "Unable to connect to database: $DBI::errstr"; + +# find a useful exp_id +$refExpID = 0; +if (1) { + # print "Exposures 10 days ago:\n"; + + my $query = "SELECT MAX(exp_id)"; + $query .= " FROM rawExp"; + $query .= " WHERE dateobs >= date_sub('$date', interval 20 day)"; + $query .= " AND dateobs <= date_sub('$date', interval 10 day)"; + + my $result = &mysql_select ($query); + + @row = $result->fetchrow_array(); + $refExpID = $row[0]; + if ($refExpID eq "") { + $refExpID = 0; + print "*** WARNING: no data in period 10-20 days before requested date (query will be slower) **\n"; + } + # print "Reference exp_id: $refExpID\n"; +} + +# check the status of summit exposures: +if (1) { + if ($verbose) { print "Exposures at summit:\n"; } + + $summitSkipped = 0; + $summitPending = 0; + +# my $query = "SELECT exp_name, exp_type, fault, state, count(*)"; + my $query = "SELECT exp_type, fault, state, count(*)"; + $query .= " FROM summitExp"; + $query .= " LEFT JOIN pzDownloadExp USING (summit_id)"; + $query .= " WHERE dateobs like '$date%'"; + $query .= " AND (state != 'drop' or state IS NULL)"; + $query .= " AND fault != 1042"; + $query .= " GROUP BY exp_type, fault, state"; + +# print "$query\n"; +# die; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s\n", "exp_type", "fault", "state", "N(exp)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d\n", $row[0], $row[1], $row[2], $row[3]; } + if ($row[2] eq "run") { + $summitPending += $row[3]; + } + if ($row[2] eq "") { + $summitSkipped += $row[3]; + } + } + if ($summitSkipped || $summitPending) { print "*** WARNING: $summitSkipped skipped rawExp, $summitPending incomplete rawExp ***\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# check for exposures not sent to chip: +if ($verbose2) { + + my $query = "SELECT summitExp.exp_name, exp_type, fault, state"; + $query .= " FROM summitExp"; + $query .= " LEFT JOIN pzDownloadExp USING (summit_id)"; + $query .= " WHERE dateobs like '$date%'"; +# $query .= " AND (state != 'drop' or state IS NULL)"; + $query .= " AND fault != 1042"; + $query .= " AND state IS NULL"; + + print "$query\n"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "OBJECT Exposures NOT sent to chipRun:\n"; + printf "%11s %15s %14s %14s\n", "exp_name", "exp_type", "fault", "state"; + $doHeader = 0; + } + printf "%11s %15s %14s %14s\n", $row[0], $row[1], $row[2], $row[3]; + } + if (! $doHeader) { print "\n"; } +} + +# check the status of raw exposures: +if (1) { + if ($verbose) { print "Exposures in rawExp:\n"; } + + $expSkipped = 0; + $expPending = 0; + + my $query = "SELECT exp_type, obs_mode, state, count(*)"; + $query .= " FROM rawExp"; + $query .= " WHERE dateobs like '$date%'"; + $query .= " AND exp_id > $refExpID"; # REF +# $query .= " AND exp_type = 'OBJECT'"; + $query .= " GROUP BY exp_type, obs_mode, state"; + $query .= " ORDER BY exp_type, obs_mode"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s\n", "exp_type", "obs_mode", "state", "N(exp)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d\n", $row[0], $row[1], $row[2], $row[3]; } + if ($row[2] eq "new") { + $expPending += $row[3]; + } + if ($row[2] eq "") { + $expSkipped += $row[3]; + } + } + if ($expSkipped || $expPending) { print "*** WARNING: $expSkipped skipped rawExp, $expPending incomplete rawExp ***\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# check the status of chip runs: +if (1) { + if ($verbose) { print "Exposures in chipRun (OBJECT, obs_mode not like ENGINEERING, not NULL):\n"; } # the obs_mode not NULL is implicit given the obs_mode not like ENGINEERING + + $chipPending = 0; + $chipSkipped = 0; + + my $query = "SELECT exp_type, obs_mode, chipRun.state, count(*)"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " GROUP BY obs_mode, chipRun.state"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s", "exp_type", "obs_mode", "state", "N(exp)\n"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d\n", $row[0], $row[1], $row[2], $row[3]; } + if ($row[2] eq "new") { + $chipPending += $row[3]; + } + if ($row[2] eq "") { + $chipSkipped += $row[3]; + } + } + if ($chipPending || $chipSkipped) { print "*** WARNING: $chipSkipped skipped chipRun, $chipPending incomplete chipRun ***\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# check the status of camera runs: +$camPending = 0; +$camSkipped = 0; +{ + if ($verbose) { print "Exposures in camRun (OBJECT, not ENGINEERING)...\n"; } + + # my $result = &mysql_select (""); + my $query = "SELECT exp_type, obs_mode, camRun.state, camProcessedExp.fault, camProcessedExp.quality, count(*)"; + $query .= " FROM rawExp join chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " GROUP BY obs_mode, camRun.state, camProcessedExp.fault, camProcessedExp.quality"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s | %5s %5s\n", "exp_type", "obs_mode", "state", "N(exp)", "fault", "quality"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d | %5d %5d\n", $row[0], $row[1], $row[2], $row[5], $row[3], $row[4]; } # exp_type, obs_mode, camRun.state, camProcessedExp.fault, camProcessedExp.quality, count(*) + if ($row[2] eq "new") { + $camPending += $row[5]; + } + if ($row[2] eq "") { + $camSkipped += $row[5]; + } + } + if ($camSkipped || $camPending) { print "*** WARNING: $camSkipped skipped camRuns, $camPending pending camRuns ***\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# check the status of fake runs: +{ + if ($verbose) { print "Exposures in fakeRun:\n"; } + + $fakePending = 0; + $fakeSkipped = 0; + + my $query = "SELECT exp_type, obs_mode, fakeRun.state, count(*)"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun using (exp_id)"; + $query .= " JOIN camRun using (chip_id)"; + $query .= " JOIN camProcessedExp using (cam_id)"; + $query .= " LEFT JOIN fakeRun using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND (camProcessedExp.quality is null or camProcessedExp.quality = 0)"; + $query .= " GROUP BY obs_mode, fakeRun.state"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s\n", "exp_type", "obs_mode", "state", "N(exp)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d\n", $row[0], $row[1], $row[2], $row[3]; } # exp_type, obs_mode, fakeRun.state, count(*) + if ($row[2] eq "new") { + $fakePending += $row[3]; + } + if ($row[2] eq "") { + $fakeSkipped += $row[3]; + } + } + if ($fakeSkipped || $fakePending) { print "$fakeSkipped skipped fakeRuns, $fakePending pending fakeRuns\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# check the status of warp runs: +{ + if ($verbose) { print "Exposures in warpRuns:\n"; } + + $warpPending = 0; + $warpSkipped = 0; + + # my $result = &mysql_select (""); + my $query = "SELECT exp_type, obs_mode, warpRun.state, count(*)"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun using (exp_id)"; + $query .= " JOIN camRun using (chip_id)"; + $query .= " JOIN camProcessedExp using (cam_id)"; + $query .= " JOIN fakeRun using (cam_id)"; + $query .= " LEFT JOIN warpRun using (fake_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND (camProcessedExp.quality IS NULL OR camProcessedExp.quality = 0)"; + $query .= " GROUP BY obs_mode, warpRun.state"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s\n", "exp_type", "obs_mode", "state", "N(exp)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d\n", $row[0], $row[1], $row[2], $row[3]; } # exp_type, obs_mode, warpRun.state, count(*) + if ($row[2] eq "new") { + $warpPending += $row[3]; + } + if ($row[2] eq "") { + $warpSkipped += $row[3]; + } + } + if ($warpSkipped || $warpPending) { print "$warpSkipped skipped warpRuns, $warpPending pending warpRuns\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# count the number of exposures per chunk, determine expected diffRuns: +{ + print "Expected number of WWdiffs excluding camRuns with bad quality:\n"; + print "NOTE: multiple exposures with same comment string (xyz visit a) are treated as 1 exposure\n"; + + # my $result = &mysql_select (""); + my $query = "SELECT Nvisit, count(*) FROM"; + $query .= " ("; + $query .= " SELECT object, filter, chunk, count(*) as Nvisit FROM"; + $query .= " ("; + $query .= " SELECT object, filter, substr(comment, 1, position(' ' in comment)) as chunk"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND obs_mode like '%SS%'"; + $query .= " AND rawExp.comment like '%visit%'"; + $query .= " AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)"; + $query .= " GROUP BY comment, filter, substr(comment, 1, position(' ' in comment))"; + $query .= " ) AS tableChunks GROUP BY object, filter, chunk"; + $query .= " ) AS tableVisits GROUP BY Nvisit"; + + my $result = &mysql_select ($query); + + $diffExpect = 0; + while (@row = $result->fetchrow_array()) { + # printf "%8s %15s : ", $row[0], $row[1]; # c, count(*) + if ($row[0] == 1) { + if ($row[1] == 1) { $verb = "quad has"; } else { $verb = "quads have"; } + printf "%3d %-11s 1 visit, no WWdiffs\n", $row[1], $verb; + } elsif ($row[0] == 2) { + if ($row[1] == 1) { $verb = "quad has"; } else { $verb = "quads have"; } + printf "%3d %-11s 2 visits, expect $row[1] WWdiff(s)\n", $row[1], $verb; + $diffExpect += $row[1]; + } elsif ($row[0] == 3) { + if ($row[1] == 1) { $verb = "quad has"; } else { $verb = "quads have"; } + printf "%3d %-11s 3 visits, expect %d WWdiffs (2 per quad)\n", $row[1], $verb, 2 * $row[1]; + $diffExpect += $row[1]*2; + } elsif ($row[0] == 4) { + if ($row[1] == 1) { $verb = "quad has"; } else { $verb = "quads have"; } + printf "%3d %-11s 4 visits, expect %d WWdiffs (2 per quad)\n", $row[1], $verb, 2 * $row[1]; + $diffExpect += $row[1]*2; + } else { + print "I don't understand, there are more visits than I expect\n"; + } + } + print "\n"; + print " * $diffExpect diffRuns expected *\n"; + if ($verbose) { print "\n"; } +} + +# List chunks with duplicate visits +{ + # my $result = &mysql_select (""); +# my $query = "SELECT Nvisit, count(*) FROM"; +# $query .= " ("; + $query .= " SELECT object, filter, chunk, Nexp_in_chunk FROM"; + $query .= " ("; + $query .= " SELECT object, filter, substr(comment, 1, position(' ' in comment)) as chunk, count(*) as Nexp_in_chunk"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND obs_mode like '%SS%'"; + $query .= " AND rawExp.comment like '%visit%'"; + $query .= " AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)"; + $query .= " GROUP BY comment, filter, substr(comment, 1, position(' ' in comment))"; + $query .= " ) AS tableChunks where (Nexp_in_chunk > 1)"; +# $query .= " ) AS tableVisits GROUP BY Nvisit"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "chunks with duplicate visits:\n"; + printf "%11s %15s %14s %5s\n", "object", "filter", "chunk", "Nexp in chunk"; + $doHeader = 0; + } + printf "%11s %15s %14s %5s\n", $row[0], $row[1], $row[2], $row[3]; + } + if (! $doHeader) { print "\n"; } +} + +# List exposures which may be used for WWdiffs +if (0) { + print "List exposures for WWdiffs excluding camRuns with bad quality:\n"; + + # my $result = &mysql_select (""); + my $query = ""; + $query .= " SELECT exp_name, object, filter, substr(comment, 1, position(' ' in comment)) as chunk"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND obs_mode like '%SS%'"; + $query .= " AND rawExp.comment like '%visit%'"; + $query .= " AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)"; + + my $result = &mysql_select ($query); + &mysql_dump_result ($result); +} + +# check the status of diff runs: +{ + if ($verbose) { print "diff processing status (not WS):\n"; } + + $diffDone = 0; + + my $query = "SELECT label, state, count(*)"; + $query .= " FROM diffRun"; + $query .= " WHERE data_group like '%$dataGroupDate'"; + $query .= " AND label NOT LIKE '%WS.nigh%'"; + $query .= " GROUP BY label, state"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%20s %6s | %8s\n", "label", "state", "N(diffs)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%20s %6s | %8d\n", $row[0], $row[1], $row[2]; } # label, state, count + if (($row[1] ne "new") && ($row[1] ne "")) { + $diffDone += $row[2]; + } + } + if ($verbose) { print "\n"; } + if ($diffDone != $diffExpect) { + print "*** WARNING: Expect $diffExpect diffRuns, complete $diffDone diffRuns: inconsistent ***\n"; + } else { + print " * $diffDone of $diffExpect diffRuns completed *\n"; + } +} + +# count diffRuns to be published: +{ + my $query = "SELECT count(*) FROM"; + $query .= " ("; + $query .= " SELECT label, state, count(*)"; + $query .= " FROM diffRun"; + $query .= " LEFT JOIN diffSkyfile using (diff_id)"; + $query .= " WHERE data_group LIKE '%$dataGroupDate%'"; + $query .= " AND label NOT LIKE '%WS.nigh%'"; + $query .= " AND (quality IS NULL OR quality = 0)"; + $query .= " GROUP BY label, state, diff_id"; + $query .= " ) as tableDiffs"; + + my $result = &mysql_select ($query); + + @row = $result->fetchrow_array(); + printf " * %d diffRuns can be published * (has at least 1 skycell with quality = 0 or quality is NULL)\n", $row[0]; + if ($verbose) { print "\n"; } + $diffRunsToBePublished = $row[0]; +} + +# check the status of publish runs: +{ + if ($verbose) { print "publishing status (not WS)...\n"; } + + $pubDone = 0; + + my $query = "SELECT diffRun.label, publishRun.state, count(*)"; + $query .= " FROM diffRun"; + $query .= " LEFT JOIN publishRun on (diff_id = stage_id)"; + $query .= " WHERE data_group LIKE '%$dataGroupDate'"; + $query .= " AND diffRun.label NOT LIKE '%WS.nigh%'"; + $query .= " GROUP BY diffRun.label, publishRun.state"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%20s %6s | %8s\n", "label", "state", "N(diffs)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%20s %6s | %8d\n", $row[0], $row[1], $row[2]; } # label, state, count + if (($row[1] ne "new") && ($row[1] ne "")) { + $pubDone += $row[2]; + } + } + if ($verbose) { print "\n"; } + if ($pubDone != $diffRunsToBePublished) { + print "*** WARNING: Expect $diffRunsToBePublished diffRuns to be published, only $pubDone pubRuns: inconsistent ***\n\n"; + } else { + print " * $pubDone of $diffRunsToBePublished diffRuns published **\n\n"; + } +} + +# check for exposures not sent to chip: +if ($verbose) { + + my $query = "SELECT exp_name, obs_mode, comment"; + $query .= " FROM rawExp left join chipRun using (exp_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND chip_id is null"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "OBJECT Exposures NOT sent to chipRun:\n"; + printf "%11s %15s %14s", "exp_name", "obs_mode", "comment\n"; + $doHeader = 0; + } + printf "%11s %15s %14s\n", $row[0], $row[1], $row[2]; + } + if (! $doHeader) { print "\n"; } +} + +# check for chipRuns not sent to camera +if ($camSkipped && $verbose) { + print "chipRuns NOT sent to camRun:\n"; + + my $query = "SELECT exp_name, obs_mode, comment, chipRun.state"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun USING (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND cam_id IS NULL"; + + my $result = &mysql_select ($query); + + printf "%11s %15s %10s %14s\n", "exp_name", "obs_mode", "chip.state", "comment"; + while (@row = $result->fetchrow_array()) { + printf "%11s %15s %10s %14s\n", $row[0], $row[1], $row[2], $row[3]; + } + print "\n"; +} + +# check for bad quality camera exposurs +if (1) { + my $query = "SELECT exp_name, camProcessedExp.quality, comment"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun USING (exp_id)"; + $query .= " JOIN camRun USING (chip_id)"; + $query .= " JOIN camProcessedExp USING (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND (camProcessedExp.quality > 0)"; + $query .= " ORDER BY comment"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + $NbadCam = 0; + + while (@row = $result->fetchrow_array()) { + if ($verbose && $doHeader) { + print "Bad quality camRun exposures:\n"; + printf "%11s %7s | %15s\n", "exp_name", "quality", "comment"; + $doHeader = 0; + } + if ($verbose) { printf "%11s %7d | %15s\n", $row[0], $row[1], $row[2]; } else { $NbadCam ++; } + } + if ($verbose && !$doHeader) { print "\n"; } + if (!$verbose) { print "$NbadCam bad quality camera exposures\n\n"; } +} + +# check the status of diff runs NOT for Solar System: +if ($verbose) { + # my $result = &mysql_select (""); + my $query = "SELECT * FROM"; + $query .= " ("; + $query .= " SELECT object, filter, chunk, count(*) as Nvisit FROM"; + $query .= " ("; + $query .= " SELECT object, filter, substr(comment, 1, position(' ' in comment)) as chunk"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND obs_mode NOT like '%SS%'"; + $query .= " AND rawExp.comment like '%visit%'"; + $query .= " AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)"; + $query .= " GROUP BY comment, filter, substr(comment, 1, position(' ' in comment))"; + $query .= " ) AS tableChunks GROUP BY object, filter, chunk"; + $query .= " ) AS tableVisits ORDER BY Nvisit, chunk, object"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "Chunks which should not generate WWdiffs (not Solar System diffs)\n"; + printf "%8s %8s %8s %8s\n", "Object", "Filter", "Chunk", "N(visit)"; + $doHeader = 0; + } + printf "%8s %8s %8s %4d\n", $row[0], $row[1], $row[2], $row[3]; + } + if (!$doHeader) { print "\n"; } +} + +# check for unpublished solar-system exposures +{ + my $query = "SELECT expall.exp_name, expall.dateobs, expall.comment, expall.warp_id, exppub.diff_id, exppub.pubstate from"; + $query .= " ("; + $query .= " SELECT exp_name, comment, dateobs, warp_id"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun using (exp_id)"; + $query .= " JOIN camRun using (chip_id)"; + $query .= " JOIN camProcessedExp using (cam_id)"; + $query .= " JOIN fakeRun using (cam_id)"; + $query .= " JOIN warpRun using (fake_id)"; + $query .= " WHERE dateobs like '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND obs_mode like '%SS%'"; + $query .= " AND (camProcessedExp.quality = 0)"; + $query .= " ) AS expall"; + $query .= " LEFT JOIN"; + $query .= " ("; + $query .= " SELECT exp_name, warpRun.warp_id, diffRun.diff_id, publishRun.state as pubstate"; + $query .= " FROM diffRun"; + $query .= " JOIN diffInputSkyfile using (diff_id)"; + $query .= " JOIN publishRun on (diffRun.diff_id = stage_id)"; + $query .= " JOIN warpRun ON ( warp1 = warp_id or warp2 = warp_id)"; + $query .= " JOIN fakeRun using (fake_id)"; + $query .= " JOIN camRun using (cam_id)"; + $query .= " JOIN camProcessedExp using (cam_id)"; + $query .= " JOIN chipRun using (chip_id)"; + $query .= " JOIN rawExp using (exp_id)"; + $query .= " WHERE diffRun.reduction like 'SWEETSPOT'"; +## $query .= " AND exp_id > $refExpID"; -- adding this restriction here slows the query way down1 + $query .= " AND stack2 IS NULL"; + $query .= " AND diffRun.data_group LIKE '%SS.$dataGroupDate'"; + $query .= " GROUP BY exp_name"; + $query .= " ) AS exppub"; + $query .= " ON expall.exp_name = exppub.exp_name"; + $query .= " WHERE exppub.exp_name is NULL"; + $query .= " OR exppub.pubstate = 'new'"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "Unpublished Solar System Exposures ( obs_mode like %SS% ) with good camera quality :\n"; + printf "%12s %12s %12s | %-19s | %-16s\n", "exp_name", "warp_id", "diff_id", "dateobs", "comment"; + $doHeader = 0; + } + if ($row[3] eq "") { $row[3] = "NULL"; } + if ($row[4] eq "") { $row[4] = "NULL"; } + printf "%12s %12s %12s | %-19s | %-16s\n", $row[0], $row[3], $row[4], $row[1], $row[2] + } +} +print "\n\n"; + +# count the number of exposures per chunk, determine expected diffRuns: +if ($verbose2) { + print "Number of exposures for each chunk:\n"; + + # my $result = &mysql_select (""); + my $query = "SELECT object, filter, chunk, count(*) as Nvisit FROM"; + $query .= " ("; + $query .= " SELECT object, filter, substr(comment, 1, position(' ' in comment)) as chunk"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND obs_mode like '%SS%'"; + $query .= " AND rawExp.comment like '%visit%'"; + $query .= " AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)"; + $query .= " GROUP BY comment, filter, substr(comment, 1, position(' ' in comment))"; + $query .= " ) AS tableChunks GROUP BY object, filter, chunk"; + + my $result = &mysql_select ($query); + + printf "%11s %7s %18s %-3s\n", "object", "filter", "chunk", "N(visit)"; + while (@row = $result->fetchrow_array()) { + printf "%11s %7s %18s %3d\n", $row[0], $row[1], $row[2], $row[3]; + } + print "\n"; +} + +exit 0; + +# global dbh? +sub mysql_select { + + my $query = $_[0]; + + my $statement = $dbh->prepare($query); + + $statement->execute(); + + return $statement; +} + +sub mysql_dump_result { + + my $result = $_[0]; + + $Nfields = $result->{NUM_OF_FIELDS}; + + print "Nfields: $Nfields\n"; + + for ($i = 0; $i < $Nfields; $i++) { + printf "Field %d : %s\n", $i, $result->{NAME}->[$i]; + } + + while (@row = $result->fetchrow_array()) { + for ($i = 0; $i < $Nfields; $i++) { + print "$row[$i] "; + } + print "\n"; + } + return; +} + +# example using the dbh prepare, execute, fetch commands: +if (0) { + + my $result = &mysql_select ("SELECT * from rawExp limit 2"); + + &mysql_dump_result ($result); + + exit 1; +} Index: branches/czw_branch/20170908/ippScripts/scripts/nightly_science.pl =================================================================== --- branches/czw_branch/20170908/ippScripts/scripts/nightly_science.pl (revision 40128) +++ branches/czw_branch/20170908/ippScripts/scripts/nightly_science.pl (revision 40483) @@ -46,5 +46,5 @@ # Grab options -my ( $date, $datetime, $camera, $dbname, $logfile, $verbose, $manual); +my ( $date, $datetime, $now, $camera, $dbname, $logfile, $verbose, $manual); my ( $help, $isburning, $force_stack_count, $force_diff_count, $force_registration, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode); my ( $registration_status, $burntool_status, $observing_status, $old_date); @@ -95,5 +95,6 @@ --verbose --force_stack_count Force the chip/warp counts. - --force_diff_count Force the chip/warp counts. + --force_diff_count Force the chip/warp counts. + --force_registration Force registration counts. --this_target_only Process only a single target. --this_filter_only Process only a single filter. @@ -255,4 +256,5 @@ $date = $datetime->ymd(); } +$now = DateTime->now(time_zone => 'UTC'); if (defined($this_target_only)) { @@ -2117,8 +2119,8 @@ month => $datetime->month, day => $datetime->day, - hour => 6, + hour => 17, # 7, minute => 30, second => 0, - time_zone => 'Pacific/Honolulu'); + time_zone => 'UTC'); foreach my $eon (keys %eon_config) { @@ -2142,6 +2144,9 @@ } } -# print "$datetime $eon_dt\n"; - if (DateTime->compare($datetime,$eon_dt) < 1) { + if ($force_registration) { + return("END_OF_NIGHT"); + } +# print "$now $eon_dt " . DateTime->compare($now,$eon_dt) . "\n"; + if (DateTime->compare($now,$eon_dt) < 1) { return("OBSERVING"); } Index: branches/czw_branch/20170908/ippScripts/scripts/rawcheck_mddb.pl =================================================================== --- branches/czw_branch/20170908/ippScripts/scripts/rawcheck_mddb.pl (revision 40483) +++ branches/czw_branch/20170908/ippScripts/scripts/rawcheck_mddb.pl (revision 40483) @@ -0,0 +1,720 @@ +#!/usr/bin/env perl + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Nebulous::Client; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Digest::MD5; +use URI; + +my $missing_tools = 0; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); + +my ($neb_server,$md_server,$dbname,$dbuser,$dbpass,$exp_id,$do_cull,$do_revalidate,$save_log); + + +my $do_ops = 1; +my $i; + +my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n"; +my $siteConfig = $ipprc->{_siteConfig}; +$dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER"); +$dbpass = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD"); + +$neb_server = $ENV{'NEB_SERVER'} unless $neb_server; + +# $dbname = 'gpc1'; +GetOptions( + 'neb_server|s=s' => \$neb_server, # This is the apache server + 'md_server|s=s' => \$md_server, # This is the actual database server + 'dbname=s' => \$dbname, + 'dbuser=s' => \$dbuser, + 'dbpass=s' => \$dbpass, + 'exp_id|x=s' => \$exp_id, + 'save_log' => \$save_log, + 'cull' => \$do_cull, + 'revalidate' => \$do_revalidate, +) || pod2usage( 2 ); + +unless(defined($do_cull)) { + $do_cull = 0; +} +unless(defined($do_revalidate)) { + $do_revalidate = 0; +} +if (($do_cull == 1)&&($do_revalidate == 1)) { + pod2usage( -msg => "Only one of --cull and --revalidate may be chosen.", -exitval => 2 ); +} + + +# Option parsing +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --neb_server (apache server)", -exitval => 2 ) + unless $neb_server; +pod2usage( -msg => "Required options: --md_server (database server)", -exitval => 2 ) + unless $md_server; +pod2usage( -msg => "Required options: --dbname", -exitval => 2 ) + unless defined $dbname; +pod2usage( -msg => "Required options: --dbuser", -exitval => 2 ) + unless defined $dbuser; +pod2usage( -msg => "Required options: --dbpass", -exitval => 2 ) + unless defined $dbpass; +pod2usage( -msg => "missing key", exitval => 2 ) + unless defined $exp_id; + +if ($save_log) { + my $logDest = "neb://any/raw_check_mddb/exp_${exp_id}"; + if ($do_cull) { + $logDest .= ".cull"; + } + elsif ($do_revalidate) { + $logDest .= ".revalidate"; + } + else { + $logDest .= ".check"; + } + + $ipprc->redirect_to_logfile($logDest) or die "Could not redirect output to logfile ${logDest}\n"; +} + +# Global options: +## Define the configuration. Ideally, this would be retrieved from the nebulous +## database, but there are some issues with that (such as grouping the b nodes +## into a less restrictive "offsite" location). + +## Set up nebulous db interface, and pull processing information to consider. +my $neb = Nebulous::Client->new( + proxy => "$neb_server", +); +die "can't connected to Nebulous Server: $neb_server" + unless defined $neb; + +## Set up md5 db interface +my $mddb = DBI->connect("DBI:mysql:database=otaMD5;host=${md_server};" . + "mysql_socket=/var/run/mysqld/mysqld.sock", + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + +## This new implementation is somewhat messy, but is more general and adaptable. +## First, we set up a requirement mapping, explaining where we want copies, and +## how many copies we want at each site. +my %requirement_map = (); +$requirement_map{ITC} = 1; +$requirement_map{OFFSITE} = 1; + +## Second, construct a list of volumes, mapped to their site location, using the +## same site locations as in the requirement map. +my %volume_map = (); +for ($i = 32; $i <= 32; $i++) { + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'ITC'; +} +for ($i = 54; $i <= 97; $i++) { + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'ITC'; +} +for ($i = 100; $i <= 126; $i++) { + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'ITC'; + $vol = sprintf("ipp%03d.1",$i); + $volume_map{$vol} = 'ITC'; +} + +for ($i = 0; $i <= 15; $i++) { + my $loc = 'OFFSITE'; +# if ($i == 6) { +# $loc = 'ITC'; +# } # This isn't "offsite", it's with the rest of the maui cluster. +# if ($i == 9) { next; } # Not online + + my $vol = sprintf("ippb%02d.0",$i); + $volume_map{$vol} = $loc; + $vol = sprintf("ippb%02d.1",$i); + $volume_map{$vol} = $loc; + if ($i <= 6) { + $vol = sprintf("ippb%02d.2",$i); + $volume_map{$vol} = $loc; + } +} + +## Next, get disk space values, and check which hosts are listed as available. +my %acceptable_volume = (); +my $mounts = $neb->mounts(); +my %volume_fill_factors = (); +my %down_volume = (); +foreach my $vol_row (@$mounts) { + my ($mount_point, $total, $used, $vol_id, $name, $host, $path, $allocate, $available, $xattr) = @{ $vol_row }; + if (($allocate == 1)&&($available == 1)&&( $used / $total < 0.98)) { + $acceptable_volume{$name} = 1; + } + else { +# print "## $name $allocate $available $used $total\n"; + $acceptable_volume{$name} = 0; + } + + if (($available != 1)&&($xattr != 5)) { + $down_volume{$name} = 1; + } + else { + $down_volume{$name} = 0; + } + + if (($name =~ /ippb05/)||($name =~ /ippb02/)) { + $acceptable_volume{$name} = 0; + } + + $volume_fill_factors{$name} = $used / $total; + + # This should prioritize hosts that we wish to deprecate. + if ($total < 52885172852) { + $volume_fill_factors{$name} = 1.0; + } +} + +## Finally, generate lists containing which volumes are located at which site. +## This allows us to randomly select a volume from the list for the site that +## we plan on replicating to. +my %volume_lists = (); +foreach my $vol_key (keys %requirement_map) { + @{ $volume_lists{$vol_key} } = grep { $acceptable_volume{$_} == 1 } ( + grep { $volume_map{$_} eq $vol_key } (keys %volume_map) + ); + print "#$vol_key " . join(' ', @{ $volume_lists{$vol_key} }) . "\n"; + + if ($#{ $volume_lists{$vol_key} } == -1) { + die "No acceptable volume found for site $vol_key!\n"; + } +} + +# die; +# Pull data from the gpc1 database about the exposure to consider + +my $verbose = 0; +my $mdcParser = PS::IPP::Metadata::Config->new; + +print("#$regtool -processedimfile -exp_id $exp_id -dbname $dbname\n"); +my $regtool_cmd = "$regtool -processedimfile -exp_id $exp_id -dbname $dbname"; +my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $regtool_cmd, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool -processedimfile: $error_code", $exp_id); +} +my $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from regtool -processedimfile", $exp_id); + +my $timer_start = time(); +my $timer = time(); +print "## rawcheck.pl: $exp_id $dbname $do_cull $do_ops $timer_start\n"; +# Do prescan checks. +if ($#{ $imfiles } > 0) { + my @to_do_imfiles = (); + my $prior_imfile; + my $append = 0; + + foreach my $imfile (@$imfiles) { + my $key = $imfile->{uri}; + my $data_state = $imfile->{data_state}; + my $md5sum = $imfile->{md5sum}; + my $hostname = $imfile->{hostname}; + my $class_id = $imfile->{class_id}; + + unless(defined($key)) { die "No database uri: $exp_id"; } + unless(defined($data_state)) { $data_state = 'no_data_state'; } + if ($data_state eq 'corrupt') { next; } # This has already been marked as bad, so we can move on. + unless(defined($md5sum)) { die "No database md5sum: $exp_id"; } + unless(defined($hostname)) { $hostname = 'ipp004'; } + unless(defined($class_id)) { die "No database class_id: $class_id"; } + + # This hash contains all the information about the instances, indexed by the full filename of the instance. + my %unified_list = (); + my %orphans = (); + + $timer = time(); + print ("\n# $key $data_state $md5sum $hostname $class_id T: $timer\n"); + + # Get nebulous results for this key, and parse them + my $stat = $neb->stat($key); + die "nebulous key: $key not found" unless $stat; + my $instances = $neb->find_instances($key, 'any'); + die "no instances found" unless $instances; + my @files = map {URI->new($_)->file if $_} @$instances; + + for (my $i = 0; $i <= $#files; $i++) { + my $f = $files[$i]; + $f =~ s%//%/%g; + my ($instance_exists,$instance_host,$instance_volume,$instance_site); + ($instance_host,$instance_volume) = parse_volume($files[$i]); + $instance_site = $volume_map{$instance_volume}; + if (-e $f) { + $instance_exists = 1; + } + else { + $instance_exists = 0; + } + $unified_list{$f}{neb_host} = $instance_host; + $unified_list{$f}{neb_vol} = $instance_volume; + $unified_list{$f}{neb_site} = $instance_site; + $unified_list{$f}{exists} = $instance_exists; + } + + # Get md database results for this exp_id/class_id + my $md_query = "select file_id,ins_id,mvol_name,site_name,file,disk_sum,(STRCMP(md5sum,disk_sum) = 0) AS is_valid FROM obj LEFT JOIN ins USING(file_id) LEFT JOIN vol USING(mvol_id) WHERE exp_id = $exp_id AND class_id = '${class_id}'"; + my $md_instances = $mddb->selectall_arrayref( $md_query ); + + $timer = time(); +# print "##NEB $timer\n"; + my $md_file_id = -1; + foreach my $r (@{ $md_instances }) { + my ($file_id,$ins_id,$vol_name,$site_name,$f,$disk_sum,$is_valid) = @{ $r }; + + if ((defined($f))&&(defined($ins_id))) { + $f =~ s%//%/%g; + unless (exists($unified_list{$f})) { + if (-e $f) { + $unified_list{$f}{exists} = 1; + } + else { + $unified_list{$f}{exists} = 0; + } + } + # This should only happen if the same scan was ingested twice. + # Be paranoid and refuse to cull if this is the case. + if (exists($unified_list{$f}{md_ins_id})) { + if (($file_id == $unified_list{$f}{md_file_id})&& + ($vol_name eq $unified_list{$f}{md_vol})&& + ($site_name eq $unified_list{$f}{md_site})&& + ($disk_sum eq $unified_list{$f}{disk_sum})&& + ($is_valid == $unified_list{$f}{is_valid})) { + print "#DUPLICATE INS: $ins_id have $unified_list{$f}{md_ins_id} already\n"; + mddb_delete_instance($mddb,$ins_id); + $do_cull = -1; + next; + } + else { + # The data is inconsistent. Make a human fix it. + print "INS ERROR: $f -> \n"; + print "\t($file_id $ins_id $vol_name $site_name $disk_sum $is_valid)\n"; + print "\t($unified_list{$f}{md_file_id} $unified_list{$f}{md_ins_id} $unified_list{$f}{md_vol} $unified_list{$f}{md_site} $unified_list{$f}{disk_sum} $unified_list{$f}{is_valid})\n"; + die; + } + } + + $unified_list{$f}{md_file_id} = $file_id; + $unified_list{$f}{md_ins_id} = $ins_id; + $unified_list{$f}{md_vol} = $vol_name; + $unified_list{$f}{md_site} = $site_name; + $unified_list{$f}{disk_sum} = $disk_sum; + $unified_list{$f}{is_valid} = $is_valid; + } + + if (defined($file_id)) { + if ($md_file_id == -1) { + $md_file_id = $file_id; + } + elsif ($file_id != $md_file_id) { + die "Instance $ins_id has bad file_id $file_id expected $md_file_id"; + } + } + } + if ($md_file_id == -1) { + # There is no md information for this exposure + $md_file_id = mddb_create_file($mddb,$key,$exp_id,$class_id,$md5sum); + $do_cull = -1; + } + + $timer = time(); +# print "##MD $timer\n"; + + # Check that evrything is consistent. + my $good_copies = 0; + my $all_copies = scalar(keys(%unified_list)); + my $a_good_copy = ''; + + my %instance_count = (); + foreach my $site (keys(%requirement_map)) { + $instance_count{$site} = 0; + } + + foreach my $f (keys(%unified_list)) { + $unified_list{$f}{good_instance} = 0; + print "$f "; + + # Identify orphans, and push their handling until later. + unless (exists($unified_list{$f}{neb_host})) { + if ($down_volume{$unified_list{$f}{md_vol}} != 1) { + # Handle this. + $orphans{$f} = $unified_list{$f}; + warn "No neb entry for $orphans{$f}{md_ins_id} $exp_id $class_id $f \n"; + } + else { + warn "No neb entry for likely due to host issue $exp_id $class_id $f \n"; + } + delete($unified_list{$f}); + next; + } + print "$unified_list{$f}{neb_host} $unified_list{$f}{neb_vol} $unified_list{$f}{neb_site} $unified_list{$f}{exists} "; + + + # Create an entry in the mddb database if that doesn't already exist. + unless (exists($unified_list{$f}{md_vol})) { + # Insert a new row in the mddb ins table for this instance. + if ($unified_list{$f}{exists} != 1) { + # Ensure that the file exists, so an md5sum can be performed. + system("touch $f"); + } + my $tmpmd5 = local_md5sum($f); + my ($ins_id, $site_name) = mddb_insert_instance($mddb,$md_file_id,$f, $unified_list{$f}{neb_vol}, $tmpmd5); + + $unified_list{$f}{md_vol} = $unified_list{$f}{neb_vol}; + $unified_list{$f}{md_site} = $site_name; + $unified_list{$f}{disk_sum} = $tmpmd5; + $unified_list{$f}{md_file_id} = $md_file_id; + $unified_list{$f}{md_ins_id} = $ins_id; + + if ($tmpmd5 ne $md5sum) { + # Insert even if we fail so we can try fixing next time. + # This may cause a problem if the first instance we checked has a bad md5sum. + warn "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + $unified_list{$f}{is_valid} = 0; + } + else { + $unified_list{$f}{is_valid} = 1; + } + } + + # This will force md5sum calculations to everything, so disk<->mddb consistency can be checked. + if ($do_revalidate) { + my $tmpmd5 = local_md5sum($f); + print "REVALIDATE: ($tmpmd5) "; + if ($tmpmd5 ne $unified_list{$f}{disk_sum}) { + mddb_update_instance($mddb,$unified_list{$f}{md_ins_id}, $tmpmd5); + warn "Updated instance $unified_list{$f}{md_ins_id} to $tmpmd5\n"; + } + if ($tmpmd5 ne $md5sum) { + warn "Revalidated md5sum does not match!"; + } + } + + print "$unified_list{$f}{md_file_id} $unified_list{$f}{md_ins_id} $unified_list{$f}{md_vol} $unified_list{$f}{md_site} $unified_list{$f}{disk_sum} $unified_list{$f}{is_valid} "; + + + if ($unified_list{$f}{exists} != 1) { + warn "Instance does not exist $exp_id $class_id $f\n"; + next; + } + if ($unified_list{$f}{neb_vol} ne $unified_list{$f}{md_vol}) { + die "Inconsistent volumes for $exp_id $class_id $f $unified_list{$f}{neb_vol} $unified_list{$f}{md_vol}\n"; + } + if ($unified_list{$f}{is_valid} != 1) { + warn "Instance is not valid $exp_id $class_id $f\n"; + next; + } + if ($unified_list{$f}{disk_sum} ne $md5sum) { + # This is a die, because if the gpc1 md5sum and the mddb md5sum don't match, there's a problem + die "Inconsistent md5sum values $exp_id $class_id $f $md5sum $unified_list{$f}{disk_sum}\n"; + } + + # If none of that stopped us, then this instance is probably fine. + $unified_list{$f}{good_instance} = 1; + $instance_count{$unified_list{$f}{neb_site}} ++; + $good_copies++; + if ($a_good_copy eq '') { + $a_good_copy = $f; + } + print "$unified_list{$f}{good_instance}"; + print "\n"; + } + + if ($good_copies == 0) { + my $orphan_copy = ''; + foreach my $orph (keys (%orphans)) { + if ($orphans{$orph}{is_valid} == 1) { + $orphan_copy = $orph; + } + } + + if ($orphan_copy eq '') { + die "No good copies available for $exp_id $class_id\n"; + } + else { + die "No good copies available for $exp_id $class_id (but $orphan_copy may exist)\n"; + } + } + + $timer = time(); +# print "##CHECK $timer\n"; + + # Attempt to repair any bad instance. + print "COUNT CHECK $good_copies $all_copies\n"; + if ($good_copies != $all_copies) { + foreach my $f (keys(%unified_list)) { + if ($unified_list{$f}{good_instance} != 1) { + system("cp $a_good_copy $f"); + my $tmpmd5 = local_md5sum($f); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + + mddb_update_instance($mddb,$unified_list{$f}{md_ins_id}, $tmpmd5); + + $do_cull = -1; + $unified_list{$f}{good_instance} = 2; + $unified_list{$f}{disk_sum} = $tmpmd5; + $instance_count{$unified_list{$f}{neb_site}} ++; + $good_copies++; + + # Print out the same information as before, but updated to show the repair. + print "$f "; + print "$unified_list{$f}{neb_host} $unified_list{$f}{neb_vol} $unified_list{$f}{neb_site} $unified_list{$f}{exists} "; + print "$unified_list{$f}{md_file_id} $unified_list{$f}{md_ins_id} $unified_list{$f}{md_vol} $unified_list{$f}{md_site} $unified_list{$f}{disk_sum} $unified_list{$f}{is_valid} "; + print "$unified_list{$f}{good_instance}"; + print "\n"; + } + } + } + + # We cannot reach this point if good_copies != all_copies. + + # Do replications + foreach my $site (keys %requirement_map) { + print "#$site $instance_count{$site} $requirement_map{$site}\n"; + if ($instance_count{$site} < $requirement_map{$site}) { + my $rep_vol = get_random_site_volume($site); + print "neb-replicate --volume $rep_vol $key\n"; + $do_cull = -1; + + if ($do_ops) { + $neb->replicate($key,$rep_vol) or die "failed to replicate $key to $rep_vol"; + if ($@) { die $@; } + + # Begin my best validation thought + system("sync") == 0 or die "Couldn't sync?"; + my $uris = $neb->find_instances($key,$rep_vol); + @$uris = map {URI->new($_)->file if $_} @$uris; + my $f = ${ $uris }[0]; + my $tmpmd5 = local_md5sum($f); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + # End my best validation thought. + + # Report on the replicated instance. + my ($ins_id, $site_name) = mddb_insert_instance($mddb,$md_file_id,$f, $rep_vol, $tmpmd5); + $unified_list{$f}{md_vol} = $rep_vol; + $unified_list{$f}{md_site} = $site_name; + $unified_list{$f}{disk_sum} = $tmpmd5; + $unified_list{$f}{is_valid} = 1; + $unified_list{$f}{md_file_id} = $md_file_id; + $unified_list{$f}{md_ins_id} = $ins_id; + + my ($instance_host,$instance_volume) = parse_volume($f); + my $instance_site = $volume_map{$instance_volume}; + $unified_list{$f}{neb_host} = $instance_host; + $unified_list{$f}{neb_vol} = $instance_volume; + $unified_list{$f}{neb_site} = $instance_site; + $unified_list{$f}{exists} = 1; + $unified_list{$f}{good_instance} = 1; + + print "$f "; + print "$unified_list{$f}{neb_host} $unified_list{$f}{neb_vol} $unified_list{$f}{neb_site} $unified_list{$f}{exists} "; + print "$unified_list{$f}{md_file_id} $unified_list{$f}{md_ins_id} $unified_list{$f}{md_vol} $unified_list{$f}{md_site} $unified_list{$f}{disk_sum} $unified_list{$f}{is_valid} "; + print "$unified_list{$f}{good_instance}"; + print "\n"; + } + } + } # End replicating + + # Do culls + if ($do_cull == 1) { + foreach my $site (keys %requirement_map) { + if ($instance_count{$site} > $requirement_map{$site}) { + my @cull_order = (); + my %sorting_hash = (); + for my $f (keys %unified_list) { + if ($unified_list{$f}{neb_site} eq $site) { + push @cull_order, $f; + $sorting_hash{$f} = $volume_fill_factors{$unified_list{$f}{neb_vol}}; + } + } + @cull_order = sort { $sorting_hash{$b} <=> $sorting_hash{$a} } @cull_order; + +# my $tmp = join ' ', @cull_order; +# print "$exp_id $site $tmp\n"; + for ($i = 0; $i < ($instance_count{$site} - $requirement_map{$site}); $i++) { + my $f = $cull_order[$i]; + my $instance_volume = $unified_list{$f}{neb_vol}; + print "neb-cull --volume $instance_volume $key\n"; + print "$f $unified_list{$f}{md_ins_id}\n"; + if ($do_ops) { + # The tilde here is to force hard volumes. Don't touch it. + # Also: the 2 is a "minimum number of copies" restriction. Let's not be crazy here. + $neb->cull($key,"~${instance_volume}",2) or die "failed to cull a superfluous instance"; + if ($@) { die "$@"; } + mddb_delete_instance($mddb,$unified_list{$f}{md_ins_id}); + } + } + } + } + + # If we're here, then we've sucessfully sorted out this instance. + foreach my $f (keys %orphans) { + print "ORPHAN $f $orphans{$f}{md_ins_id}\n"; + mddb_delete_instance($mddb,$orphans{$f}{md_ins_id}); +# unlink($f) or warn "Could not unlink $f: $!"; + } + + } # End culling + + } # End scan over imfiles +} # End check for gpc1 database entry +# End prescan checks. +if ($do_cull == -1) { + die "Could not cull after repairing a file.\n"; +} + +# mddb_create_file($mddb,$key,$exp_id,$class_id,$md5sum); +sub mddb_create_file { + my $db = shift; + my $key = shift; + my $exp_id = shift; + my $class_id = shift; + my $md5sum = shift; + + my $query = "INSERT INTO obj (neb_key, exp_id, class_id, md5sum) VALUES (?, ?, ?, ?)"; + $db->do("BEGIN"); + my $stmt = $db->prepare($query); + $stmt->execute($key,$exp_id,$class_id,$md5sum) or + &my_die("failed to insert obj with new file $key $exp_id $class_id $md5sum"); + $stmt->finish(); + $db->do("COMMIT"); + + $query = "SELECT file_id FROM obj WHERE exp_id = $exp_id AND class_id = '$class_id'"; + my $md_instances = $mddb->selectall_arrayref( $query ); + my $row = ${ $md_instances }[0]; + my $file_id = ${ $row }[0]; + return($file_id); +} + +sub mddb_update_instance { + my $db = shift; + my $ins_id = shift; + my $disk_sum = shift; + + my $query = "UPDATE ins SET disk_sum = ? WHERE ins_id = ?"; + $db->do("BEGIN"); + my $stmt = $db->prepare($query); + $stmt->execute($disk_sum,$ins_id) or + &my_die("failed to update ins with new sum $ins_id $disk_sum"); + $stmt->finish(); + $db->do("COMMIT"); +} + +sub mddb_insert_instance { + my $db = shift; + my $file_id = shift; + my $file = shift; + my $volume = shift; + my $disk_sum = shift; + + my $query = "INSERT INTO ins (file_id, file, mvol_id, disk_sum) SELECT ?, ?, mvol_id, ? FROM vol WHERE vol.mvol_name = ?"; + $db->do("BEGIN"); + my $stmt = $db->prepare($query); + $stmt->execute($file_id,$file,$disk_sum,$volume) or + &my_die("failed to insert ins with new instance $file_id $file $volume $disk_sum"); + $stmt->finish(); + $db->do("COMMIT"); + + $query = "SELECT ins_id, site_name FROM ins JOIN vol USING(mvol_id) WHERE file_id = $file_id AND file = '$file'"; + my $md_instances = $mddb->selectall_arrayref( $query ); + my $row = ${ $md_instances }[0]; + my $ins_id = ${ $row }[0]; + my $site_name = ${ $row }[1]; + return($ins_id, $site_name); +} + +sub mddb_delete_instance { + my $db = shift; + my $ins_id = shift; + + print "DELETE $ins_id\n"; + my $query = "DELETE FROM ins WHERE ins_id = ?"; + $db->do("BEGIN"); + my $stmt = $db->prepare($query); + $stmt->execute($ins_id) or + &my_die("failed to delete ins entry $ins_id"); + $stmt->finish(); + $db->do("COMMIT"); +} + +sub local_md5sum { + my $filename = shift; + my $volume = (split /\//, $filename)[2]; + my $host = $volume; + $host =~ s/\.\d//; +# print "$filename $host $volume\n"; + my $response = `ssh $host remote_md5sum.pl $filename`; + chomp($response); + my ($sum, undef) = split /\s+/, $response; + unless(defined($sum)) { + my_die("Failed to calculate md5sum locally. $filename $host $volume"); + } + return($sum); +} + +sub parse_volume { + my $filename = shift(@_); + my $full_volume = (split /\//, $filename)[2]; + my ($hostname,undef) = split /\./, $full_volume; # /; + return($hostname,$full_volume); +} + +sub get_random_site_volume { + my $site_key = shift(@_); + my $NN = scalar @{ $volume_lists{$site_key} }; + my $backup_volume = ${ $volume_lists{$site_key} }[int(rand($NN))]; + return($backup_volume); +} + +sub my_die { + my $msg = shift(@_); + print $msg . "\n"; + foreach my $a (@_) { + print "ARG: $a\n"; + } + die; +} + + + + + + + + + + + + + + + + + + + + +__END__ + +