Index: /branches/czw_branch/20170908/tools/diskspace/rework/result2size.pl =================================================================== --- /branches/czw_branch/20170908/tools/diskspace/rework/result2size.pl (revision 40475) +++ /branches/czw_branch/20170908/tools/diskspace/rework/result2size.pl (revision 40476) @@ -1,3 +1,3 @@ -#! /usr/local/bin/perl -w +#!/usr/bin/perl -w my %sizes = ( @@ -114,4 +114,11 @@ %counts_full = (); %counts_total = (); + +foreach my $stage_name ('raw','chip','chip_lies','cam','warp','warp_lies','diff','diff_lies','stack','ssky','skycal','ff') { + $sums{$stage_name} = 0; + $counts_full{$stage_name} = 0; + $counts_total{$stage_name} = 0; +} + while (<>) { chomp; Index: /branches/czw_branch/20170908/tools/fix.ws.nightlyscience =================================================================== --- /branches/czw_branch/20170908/tools/fix.ws.nightlyscience (revision 40476) +++ /branches/czw_branch/20170908/tools/fix.ws.nightlyscience (revision 40476) @@ -0,0 +1,125 @@ +#!/bin/csh -f + +if ($#argv != 2) then + echo "USAGE: fix.ws.nightlyscience (password) (mode)" + echo "MODES: help, list, update, difflist, difflogs, difffix" + echo "give password for ippuser account" + exit 2 +endif + +set pword = $1 +set mode = $2 + +set thisyear = "2015" +set realyear = `date +%Y` + +if ($thisyear != $realyear) then + echo "WARNING: the hardwired year is $thisyear, but it seems we are past $thisyear. if all $thisyear entries are done, please update the year" + echo " (we have a hardwired year to avoid excessively long queries)" +endif + +if ($mode == "help") then + echo "This script provides tools to manage the WS diffs which are stuck due to missing inputs" + echo "USAGE: fix.ws.nightlyscience (password) (mode)" + echo "There are 6 mode options:" + echo " help: this help info" + echo " list: display WS diff inputs which are not ready for processing" + echo " update: provide chiptool and warptool commands to set the input states to update" + echo " NOTE: the above set the label to 'ws_nightly_update', which is included by default in stdscience" + echo " difflist: list outstanding diffRuns" + echo " difflogs: select logs from the outstanding diffRuns and pipe through 'less'" + echo " difffix: provide difftool commands to set the fault and quality for the faulted diff runs (fault 2 -> 66, 5 -> 42)" + exit 1 +endif + +if ($mode == "list") then + set query = "" + set query = "$query select chip_id, warp_id, diff_id, diffRun.data_group, chipRun.state, warpRun.state, diffRun.state" + set query = "$query from diffRun" + set query = "$query join diffInputSkyfile using (diff_id)" + set query = "$query join warpRun on (warp1 = warp_id)" + set query = "$query join fakeRun using (fake_id)" + set query = "$query join camRun using (cam_id)" + set query = "$query join chipRun using (chip_id)" + set query = "$query where diffRun.state = 'new'" + set query = "$query and diffRun.label like '%WS.nightlyscience'" + set query = "$query and diffRun.data_group like '%.$thisyear%'" + set query = "$query group by chip_id" + + mysql -h scidbm -u ippuser -p$pword gpc1 -e "$query" + exit 0 +endif + +if ($mode == "update") then + set query = "" + set query = "$query select chip_id, warp_id, diff_id, diffRun.data_group, chipRun.state, warpRun.state, diffRun.state" + set query = "$query from diffRun" + set query = "$query join diffInputSkyfile using (diff_id)" + set query = "$query join warpRun on (warp1 = warp_id)" + set query = "$query join fakeRun using (fake_id)" + set query = "$query join camRun using (cam_id)" + set query = "$query join chipRun using (chip_id)" + set query = "$query where diffRun.state = 'new'" + set query = "$query and diffRun.label like '%WS.nightlyscience'" + set query = "$query and diffRun.data_group like '%.$thisyear%'" + + set qchip = "$query and (chipRun.state != 'full')" + set qwarp = "$query and (warpRun.state != 'full')" + + set qchip = "$qchip group by chip_id" + set qwarp = "$qwarp group by chip_id" + + mysql -B -h scidbm -u ippuser -p$pword gpc1 -e "$qchip" | awk '(NR > 1){printf "chiptool -dbname gpc1 -setimfiletoupdate -set_label ws_nightly_update -chip_id %s\n", $1}' + mysql -B -h scidbm -u ippuser -p$pword gpc1 -e "$qwarp" | awk '(NR > 1){printf "warptool -dbname gpc1 -setskyfiletoupdate -set_label ws_nightly_update -warp_id %s\n", $2}' + exit 0 +endif + +if (($mode == "difflist") || ($mode == "diffstate")) then + set query = "" + set query = "$query SELECT diffSkyfile.diff_id, diffSkyfile.skycell_id, diffRun.label, diffRun.data_group, diffRun.state, diffSkyfile.fault" + set query = "$query FROM diffRun" + set query = "$query JOIN diffSkyfile using (diff_id)" + set query = "$query WHERE diffSkyfile.fault != 0" + set query = "$query AND diffRun.label like '%.WS.nightlyscience'" + + mysql -h scidbm -u ippuser -p$pword gpc1 -e "$query" + exit 0 +endif + +if ($mode == "difflogs") then + set query = "" + set query = "$query SELECT diffSkyfile.path_base" + set query = "$query FROM diffRun" + set query = "$query JOIN diffSkyfile using (diff_id)" + set query = "$query WHERE diffSkyfile.fault != 0" + set query = "$query AND diffRun.label like '%.WS.nightlyscience'" + + foreach f (`mysql -B -h scidbm -u ippuser -p$pword gpc1 -e "$query" | awk '(NR > 1){printf "%s.log\n", $1}'`) + set file = `neb-locate -p $f` + if ($status) then + echo "problem finding log file $f, here is neb-stat output:" + neb-stat $f + echo ----------------------------- + echo "press cntl-D" + cat + continue + endif + less $file + end + exit 0 +endif + +if ($mode == "difffix") then + set query = "" + set query = "$query SELECT diffSkyfile.diff_id, diffSkyfile.skycell_id, diffRun.label, diffRun.data_group, diffRun.state, diffSkyfile.fault" + set query = "$query FROM diffRun" + set query = "$query JOIN diffSkyfile using (diff_id)" + set query = "$query WHERE diffSkyfile.fault != 0" + set query = "$query AND diffRun.label like '%.WS.nightlyscience'" + + mysql -B -h scidbm -u ippuser -p$pword gpc1 -e "$query" | awk '($6 == 2){printf "difftool -dbname gpc1 -updatediffskyfile -fault 0 -set_quality 66 -diff_id %s -skycell_id %s\n", $1, $2}' + mysql -B -h scidbm -u ippuser -p$pword gpc1 -e "$query" | awk '($6 == 5){printf "difftool -dbname gpc1 -updatediffskyfile -fault 0 -set_quality 42 -diff_id %s -skycell_id %s\n", $1, $2}' + exit 0 +endif + +exit 2 Index: /branches/czw_branch/20170908/tools/regpeek.pl =================================================================== --- /branches/czw_branch/20170908/tools/regpeek.pl (revision 40475) +++ /branches/czw_branch/20170908/tools/regpeek.pl (revision 40476) @@ -21,6 +21,6 @@ } print "Date: $date\n"; -print "CMD: regtool -checkstatus -date $date -class_id ota33 -dbname gpc1 -simple\n"; -chomp(my $bad_exp = `regtool -checkstatus -date $date -class_id ota33 -dbname gpc1 -simple | grep 'stop run' | head -1`); +print "CMD: regtool -checkstatus -date $date -class_id ota33 -dbname $dbname -simple\n"; +chomp(my $bad_exp = `regtool -checkstatus -date $date -class_id ota33 -dbname $dbname -simple | grep 'stop run' | head -1`); my @row = split /\s+/, $bad_exp; if ($#row == 18) { @@ -49,5 +49,5 @@ print "$exp_name $class_id $bt_state $data_state $uri\t"; if (($bt_state > -10)&&($data_state eq 'check_burntool')) { - print "#??? regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state pending_burntool -dbname gpc1"; + print "#??? regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state pending_burntool -dbname $dbname"; } print "\n"; Index: /branches/czw_branch/20170908/tools/seed_summitExp.pl =================================================================== --- /branches/czw_branch/20170908/tools/seed_summitExp.pl (revision 40476) +++ /branches/czw_branch/20170908/tools/seed_summitExp.pl (revision 40476) @@ -0,0 +1,124 @@ +#! /usr/bin/env perl + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + + +# Tool to seed the summitExp table with an initial exposure, to give the automated summit copy tools a starting point. + +my $missing_tools = 0; +my $dsprodls = can_run('dsproductls') or (warn "Can't find dsproductls" and $missing_tools = 1); +my $pztool = can_run('pztool') or (warn "Can't find pztool" and $missing_tools = 1); + +if ($missing_tools != 0) { + die "Required tools not available"; +} + +my ($dbname, $dbuser, $dbpass); +my ($datastore_uri, $camera, $telescope); +my ($exp_name); + +# Get config information +my $ipprc = PS::IPP::Config->new( ) or die "Could not create config object.\n"; +my $siteConfig = $ipprc->{_siteConfig}; +$dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER"); +$dbpass = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD"); + +GetOptions( + 'dbname=s' => \$dbname, + 'dbuser=s' => \$dbuser, + 'dbpass=s' => \$dbpass, + 'exp_name=s' => \$exp_name, + ) || pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dbname", -exitval => 2 ) + unless defined $dbname; +pod2usage( -msg => "Required options: --dbuser", -exitval => 2 ) + unless defined $dbuser; +pod2usage( -msg => "Required options: --dbpass", -exitval => 2 ) + unless defined $dbpass; +pod2usage( -msg => "Required option Last exp_name to skip: --exp_name", -exitval => 2) + unless defined $exp_name; + +my $mdcParser = PS::IPP::Metadata::Config->new; + +# Get datastore information +my $pz_cmd = "$pztool -datastore -dbname $dbname"; + +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $pz_cmd, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die "Failed to run pztool to determine datastore uri"; +} + +my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + die "Unable to determine datastore uri from pztool output."; +my $metadata = parse_md_list($MDlist); +my $ds_entry = $metadata->[0]; + +$datastore_uri = $ds_entry->{uri}; +$camera = $ds_entry->{camera}; +$telescope = $ds_entry->{telescope}; + +# Get all entries after the one we want to ignore + +my $ds_cmd = "$dsprodls --uri $datastore_uri --last_fileset $exp_name"; +( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $ds_cmd, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die "Failed to run dsproductls to get results from datastore $datastore_uri\n"; +} + + +my @rows = split /\n/, (join "", @$stdout_buf); + +my $have_an_entry = 0; +foreach my $ds_entry (@rows) { + if ($ds_entry =~ /^#/) { next; } + if ($have_an_entry == 1) { next; } + + my ($new_exp_uri, $new_exp_name, $new_exp_dateobs, $new_exp_type) = split /\s+/, $ds_entry; + + unless ((defined($new_exp_uri))&&(defined($new_exp_name))&&(defined($new_exp_dateobs))&&(defined($new_exp_type))) { + next; + } + unless (($new_exp_uri ne '')&&($new_exp_name ne '')&&($new_exp_dateobs ne '')&&($new_exp_type ne '')) { + next; + } + $have_an_entry = 1; + + # This is formally done by psTimeFromISO, but this should be fine for seeding + $new_exp_dateobs =~ s/T/ /; + $new_exp_dateobs =~ s/Z//; + + my $insert_query = "INSERT INTO summitExp (summit_id, exp_name, camera, telescope, dateobs, exp_type, uri, imfiles, fault, epoch) VALUES ("; + $insert_query .= " NULL, "; # summit_id + $insert_query .= " '$new_exp_name', "; # exp_name + $insert_query .= " '$camera', "; # camera + $insert_query .= " '$telescope', "; # telescope + $insert_query .= " '$new_exp_dateobs', "; # dateobs + $insert_query .= " '$new_exp_type', "; # exp_type + $insert_query .= " '$new_exp_uri', "; # uri + $insert_query .= " NULL, "; # imfiles + $insert_query .= " 0, "; #fault + $insert_query .= " NULL); "; #epoch + + print "BEGIN;\n"; + print "$insert_query\n"; + print "COMMIT;\n"; + +} + + Index: /branches/czw_branch/20170908/tools/stsci_disks/p4_restore.pl =================================================================== --- /branches/czw_branch/20170908/tools/stsci_disks/p4_restore.pl (revision 40475) +++ /branches/czw_branch/20170908/tools/stsci_disks/p4_restore.pl (revision 40476) @@ -29,5 +29,7 @@ my ( $db, $dbhost, $dbpass, $dbuser); -my $conffile = "/home/panstarrs/ipp/ippconfig/nebulous.config"; +#my $conffile = "/home/panstarrs/ipp/ippconfig/nebulous.config"; +my $conffile = '/data/ippc64.1/ippitc/ippconfig/nebulous.config'; + if (-e $conffile) { # printf STDERR "found config\n"; @@ -53,5 +55,5 @@ pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless - defined $host && defined $volume && defined $dir && + defined $host && defined $volume && defined $dir && defined $db && defined $dbhost && defined $dbuser && defined $dbpass; @@ -71,21 +73,36 @@ my $time = time(); -my $dbh = DBI->connect( - "DBI:mysql:database=$db:host=$dbhost", - $dbuser, - $dbpass, - { - RaiseError => 1, - PrintError => 0, - AutoCommit => 1, - }, - ); + +#$dbhost = '166.122.172.39'; +#my $dbport = '3308'; +#my $dbh = DBI->connect( +# "DBI:mysql:database=$db:host=$dbhost:port=$dbport", +# $dbuser, +# $dbpass, +# { +# RaiseError => 1, +# PrintError => 0, +# AutoCommit => 1, +# }, +# ); +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + +my $dbh = DBI->connect("DBI:mysql:database=${db};host=${dbhost};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; -my $logDest = "neb://any/stsci_shuffle/restore/${host}.${volume}/${source}.${volume}/${dir}.log.${time}"; +my $logDest = "/export/${host}.${volume}/retire_rsync/restore/${host}.${volume}/${source}.${volume}/${dir}.log.${time}"; + +#if ((${host} eq 'ippb04')&&($volume == 2)) { +# $logDest = "/export/${host}.0/stsci_shuffle/restore/${host}.${volume}/${source}.${volume}/${dir}.log.${time}"; +#} + my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n"; $ipprc->redirect_output($logDest) or die "Could not redirect output to logfile ${logDest}\n"; -my $root = "/export/${host}.${volume}/stsci_shuffle/${source}.${source_vol}/nebulous/${dir}"; +my $root = "/export/${host}.${volume}/retire_rsync/data/${source}.${source_vol}/nebulous/${dir}"; my @subdirectories = <$root/*>; @@ -113,5 +130,5 @@ my $destination_subdir = $subdir; - $destination_subdir =~ s%/stsci_shuffle/${source}.${source_vol}/nebulous%/nebulous%; + $destination_subdir =~ s%/retire_rsync/data/${source}.${source_vol}/nebulous%/nebulous%; my @files = <${subdir}/*>; @@ -121,49 +138,51 @@ print "LOG BEGIN $subdir $time $dtime\n"; for (my $i = 0; $i <= $#files; $i++) { - my $f = $files[$i]; + my $f = $files[$i]; my ($ins_id); $ins_id = $f; $ins_id =~ s/^$subdir\///; $ins_id =~ s/\..*//; - - # Write log of what we plan to do. - my $destination_f = $f; - $destination_f =~ s%/stsci_shuffle/${source}.${source_vol}/nebulous%/nebulous%; - - my $db_uri = $destination_f; - $db_uri =~ s%^/export/%/data/%; - $db_uri = 'file://' . $db_uri; - print "$ins_id $f $destination_f $db_uri\n"; + # Write log of what we plan to do. + my $destination_f = $f; + $destination_f =~ s%/retire_rsync/data/${source}.${source_vol}/nebulous%/nebulous%; -# die; + my $db_uri = $destination_f; + $db_uri =~ s%^/export/%/data/%; + $db_uri = 'file://' . $db_uri; - # Do move of file. - if (-e $destination_f) { - print "FILE ALREADY FOUND\n"; - next; - } + print "$ins_id $f $destination_f $db_uri\n"; - my $validate_ins_id_query = "SELECT * FROM instance WHERE vol_id = ${source_vol_id} AND ins_id = ${ins_id}"; - my @instances = $dbh->selectall_arrayref( $validate_ins_id_query ); - if ($#instances != 0) { # We expect one instance on the source - print "NO DB ENTRY\n"; - next; - } -# print "PASSED DB CHECK: $#instances\n"; -# system("stat ${f}"); - system("mv ${f} ${destination_f}"); -# system("stat ${destination_f}"); +# die; - # Register database change: + # Do move of file. + if (-e $destination_f) { + print "FILE ALREADY FOUND\n"; + next; + } - unless (-e $destination_f) { - die; - } - - $query = $dbh->prepare( $sql->update_instance_uri ); - $query->execute( $local_vol_id, $db_uri, $ins_id ); - $query->finish; -# die; + my $validate_ins_id_query = "SELECT * FROM instance WHERE vol_id = ${source_vol_id} AND ins_id = ${ins_id}"; + my @instances = $dbh->selectall_arrayref( $validate_ins_id_query ); + if ($#instances != 0) { # We expect one instance on the source + print "NO DB ENTRY\n"; + next; + } + + +# print "PASSED DB CHECK: $#instances\n"; +# system("stat ${f}"); + system("mv ${f} ${destination_f}"); +# system("stat ${destination_f}"); + + # Register database change: + + unless (-e $destination_f) { + die; + } + + $query = $dbh->prepare( $sql->update_instance_uri ); + $query->execute( $local_vol_id, $db_uri, $ins_id ); + $query->finish; +# die; } # END loop over files rmdir($subdir);