Index: trunk/ippScripts/scripts/destreak_restore_camera.pl =================================================================== --- trunk/ippScripts/scripts/destreak_restore_camera.pl (revision 33602) +++ trunk/ippScripts/scripts/destreak_restore_camera.pl (revision 33618) @@ -21,4 +21,5 @@ use File::Temp qw( tempfile ); use File::Basename qw( basename dirname ); +use File::Copy; use PS::IPP::Metadata::Config; use PS::IPP::Metadata::List qw( parse_md_list ); @@ -33,6 +34,8 @@ my $missing_tools; my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); -my $isdestreaked = can_run('isdestreaked') or (warn "Can't find isdestreaked" and $missing_tools = 1); -my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $checkfits = can_run('checkfits') or (warn "Can't find checkfits" and $missing_tools = 1); +my $ftable = can_run('ftable') or (warn "Can't find ftable" and $missing_tools = 1); +my $censorObjects = can_run('censorObjects') or (warn "Can't find censorObjects" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -41,5 +44,5 @@ # Parse the command-line arguments -my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base); +my ($magic_ds_id, $camera, $streaks, $cam_id, $uri, $path_base); my ($outroot, $release, $bytes, $md5sum); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); @@ -47,7 +50,6 @@ GetOptions( 'magic_ds_id=s' => \$magic_ds_id,# Magic destreak run identifier - 'camera=s' => \$camera, # camera for evaluating file rules - 'stage=s' => \$stage, # raw, chip, warp, or diff - 'stage_id=s' => \$stage_id, # exp_id, chip_id, warp_id, or diff_id + 'camera=s' => \$camera, # path_base of the input + 'cam_id=s' => \$cam_id, 'path_base=s' => \$path_base, # path_base of the input 'dbname=s' => \$dbname, # Database name @@ -59,32 +61,25 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --magic_ds_id --camera --stage --stage_id --component --path_base", +pod2usage( -msg => "Required options: --magic_ds_id --camera --cam_id --path_base", -exitval => 3) unless defined $magic_ds_id and defined $camera and - defined $stage and - defined $stage_id and + defined $cam_id and defined $path_base; -$component = "exposure"; - if ($camera ne 'GPC1') { - my_die("camera must be GPC1 not $camera ", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); -} - -if ($stage ne 'camera') { - my_die("stage must be camera", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); -} - -my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration - -$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile; + my_die("camera must be GPC1 not $camera ", $magic_ds_id, $PS_EXIT_PROG_ERROR); +} + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile; my $nebulousServer = $ENV{NEB_SERVER}; -&my_die("cannot find NEB_SERVER in environment", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer; +&my_die("cannot find NEB_SERVER in environment", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer; my $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); }; if ($@ or not defined $nebulous) { - &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); + &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); } @@ -95,5 +90,5 @@ my $nebulousInput = inNebulous($dirname); -&my_die ("$dirname is not in nebulous", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) unless $nebulousInput; +&my_die ("$dirname is not in nebulous", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) unless $nebulousInput; my $backup_path_base = $dirname . "/SR_" . "$basename"; @@ -103,25 +98,41 @@ my $bastrom = $ipprc->filename("PSASTRO.OUTPUT", $backup_path_base); -print "swap $bastrom and $astrom\n"; restore_file($astrom, $bastrom, 1) or - &my_die("failed to restore smf file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); + &my_die("failed to restore smf file", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + +# Now restore the mask files my @otas = qw(XY01 XY02 XY03 XY04 XY05 XY06 XY10 XY11 XY12 XY13 XY14 XY15 XY16 XY17 XY20 XY21 XY22 XY23 XY24 XY25 XY26 XY27 XY30 XY31 XY32 XY33 XY34 XY35 XY36 XY37 XY40 XY41 XY42 XY43 XY44 XY45 XY46 XY47 XY50 XY51 XY52 XY53 XY54 XY55 XY56 XY57 XY60 XY61 XY62 XY63 XY64 XY65 XY66 XY67 XY71 XY72 XY73 XY74 XY75 XY76); -# go fix the mask files foreach my $class_id (@otas) { my $mask = "$path_base.$class_id.mk.fits"; my $bmask = "$backup_path_base.$class_id.mk.fits"; my $rmask = "$recovery_path_base.$class_id.mk.fits"; - restore_file($mask, $bmask, 0); -# or &my_die("failed to restore $mask", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); + if (!restore_file($mask, $bmask, 0)) { + # no good mask file. Mark any chips that need this mask file to be useful as bad + # XXX: what about chipRuns with multiple camRuns can't we preserve those? + my $command = "$chiptool -dropprocessedimfile -cam_id $cam_id -class_id $class_id -set_quality 45"; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool -addresult: $error_code", $magic_ds_id, + $error_code); + } + } else { + print "Skipping command: $command\n"; + } + } } { - # this sets the camRun's magicked value back to zero + # this sets the camRun's magicked value back to zero and deletes the magicDSFile + # row from the database my $command = "$magicdstool -revertdestreakedfile -i_am_sure"; $command .= " -state goto_restored"; $command .= " -magic_ds_id $magic_ds_id"; - $command .= " -component $component"; + $command .= " -component exposure"; $command .= " -dbname $dbname" if defined $dbname; @@ -132,5 +143,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool -addresult: $error_code", $magic_ds_id, $component, + &my_die("Unable to perform magicdstool -addresult: $error_code", $magic_ds_id, $error_code); } @@ -139,7 +150,28 @@ } } +{ + # this sets the magicDSRun.state to restored + my $command = "$magicdstool -updaterun"; + $command .= " -set_state restored"; + $command .= " -magic_ds_id $magic_ds_id"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool -addresult: $error_code", $magic_ds_id, + $error_code); + } + } else { + print "Skipping command: $command\n"; + } +} exit 0; ### Pau. + sub restore_file { my $original_url = shift; @@ -147,5 +179,15 @@ my $smf = shift; - my $original = check_file($original_url, 1); + print "\nStarting restore for $original_url\n"; + + # It is the backup that we expect to want to use, check it first. + my $backup = check_file($backup_url, 1, $smf); + + # Is backup a good file whether censored or not? + my $backup_ok = ($backup->{so_exists} and $backup->{num_instances}); + + # Check original, skip full check if backup is ok and uncensored since the backup + # is the one that we want to keep. + my $original = check_file($original_url, (!$backup_ok or $backup->{censored}), $smf); my $swap_urls = 0; @@ -153,62 +195,70 @@ my $replicate = 0; - if ($original->{so_exists}) { - if ($original->{num_instances} == 0 or $original->{censored}) { - # expected case: censored original - # check the status of the backup - my $backup = check_file($backup_url, 1); - if ($backup->{so_exists}) { - if (!$backup->{censored}) { - # expected case backup is uncensored - if ($backup->{inconsistent}) { - if (!repair_file($backup), $smf) { - print STDERR "failed to repair uncensored backup file $backup_url\n"; - return 0; - } - } - # All systems go + if ($backup_ok and !$backup->{censored}) { + # + # Normal case. A good uncensored backup file is available + # + if ($backup->{inconsistent}) { + # At least one of the instances was bad. Repair. + repair_file($backup, $smf) or + &my_die( "failed to repair $backup_url", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + } + + $replicate = $backup->{num_instances} < 2; + + if ($original->{so_exists}) { + # + # Normal case. + # + $swap_urls = 1; + $kill_censored = 1; + } else { + # no file at original url rename the backup + # note: rename_file prints a message and dies on failure + rename_file($backup_url, $original_url); + } + } else { + # Something is wrong with the backup file, either not ok or censored + # check the original + my $original_ok = ($original->{so_exists} and $original->{num_instances}); + if ($original_ok) { + if ($original->{inconsistent}) { + repair_file($original, $smf) or + &my_die( "failed to repair $original_url", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + } + if (!$original->{censored}) { + # Note this is where we end up if this file has already been restored + print "Keeping uncensored file at original url: $original_url\n"; + } else { + print "No uncensored backup. Keeping censored file at original url: $original_url\n"; + } + $kill_censored = $backup->{so_exists}; + $replicate = $original->{num_instances} < 2; + } else { + # Original is bad. Is the backup at least a good file. If so swap it back in even if it is censored. + if ($backup_ok) { + print "No file found at $original_url\n"; + print "Swapping in $backup_url\n"; + if ($backup->{inconsistent}) { + repair_file($backup, $smf) or + &my_die( "failed to repair $backup_url", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + } + $replicate = $backup->{num_instances} < 2; + if ($original->{so_exists}) { $swap_urls = 1; $kill_censored = 1; - $replicate = $backup->{num_instances} < 2; } else { - # user intervention required - print STDERR "both original and backup url point to censored file names\n"; - my $name = $original->{instances}->[0]; - print STDERR " original: $name\n"; - $name = $backup->{instances}->[0]; - print STDERR " backup: $name\n"; - return 0; + # note: rename_file prints a message and dies on failure + rename_file($backup_url, $original_url); } } else { - print STDERR "no backup file exists: $backup_url\n"; + # we're out of luck + print "No good file for $original_url found\n"; return 0; } - } else { - # original exists but does not appear to be censored. Just have a quick look at the backup - my $backup = check_file($backup_url, 0); - if ($backup->{so_exists} and $backup->{num_instances} and !$backup->{censored}) { - # user intervention required - print STDERR "both original and backup url point to un-censored file names\n"; - my $name = $original->{instances}->[0]; - print STDERR " original: $name\n"; - $name = $backup->{instances}->[0]; - print STDERR " backup: $name\n"; - return 0; - } - if ($original->{inconsistent} or scalar @{$file->{bad_instances}} > 0) { - if (!repair_file($original, $smf)) { - print STDERR "failed to repair uncensored original file $original_url\n"; - return 0; - } - } - $kill_censored = 1; - $replicate = $original->{num_instances} < 2; - } - } else { - # no original storage object exists - # We could check for a backup and rename it but this should be a very unusual case - print STDERR "No storage object found for $original_url\n"; - return 0; - } + } + } + + # Ok Now we know what we need to do so do it if (!$no_op) { @@ -217,5 +267,5 @@ print "failed to swap $backup_url\n"; print " to $original_url\n"; - &my_die( "aborting", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); + &my_die( "aborting", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); } else { print "restored $original_url\n"; @@ -227,7 +277,7 @@ } - if (!$replicate) { + if ($replicate) { $ipprc->replicate_file($original_url) - or &my_die( "failed to replicate $original_url", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); + or &my_die( "failed to replicate $original_url", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); } } else { @@ -244,4 +294,5 @@ my $url = shift; my $full_check = shift; + my $smf = shift; my %file; @@ -260,5 +311,5 @@ my $stat = $nebulous->stat($url); - &my_die("nebulous stat failed for $url", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) unless $stat; + &my_die("nebulous stat failed for $url", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) unless $stat; $file{total_instances} = $stat->[7]; @@ -270,17 +321,17 @@ }; if (!$instances or !scalar @$instances) { + print "No instances found for $url\n"; return \%file; } - my $first_inst; - my $first_size = 0; - my $first_md5 = ""; - my $first_censored ; - my $available_instances = 0; + my $good_inst; + my $good_size = 0; + my $good_md5 = ""; + my $good_censored ; foreach my $inst (@$instances) { $inst =~ s/file\:\/\///g; if ( ! -e $inst) { - print STDERR "instance file $inst does not exist\n"; + print "instance file $inst does not exist\n"; push @{$file{bad_instances}}, $inst; next; @@ -288,31 +339,95 @@ my $this_size = -s $inst; if (!$this_size) { - print STDERR "instance file $inst has zero size\n"; + print "instance file $inst has zero size\n"; push @{$file{bad_instances}}, $inst; next; } - $available_instances++; - push @{$file{instances}}, $inst; - my $this_md5 = $full_check ? file_md5_hex($inst) : ""; - if (!defined $first_inst) { - $first_inst = $inst; - $first_size = $this_size; - $first_md5 = $this_md5; - $first_censored = ($inst =~ /SR_/); - # unless full_check all we really care about is the censored status + + my $this_censored = ($inst =~ /SR_/); + + my $this_md5 = ""; + if ($full_check) { + $this_md5 = file_md5_hex($inst); + if ($smf) { + my $command = "$ftable -list $inst | grep psf | wc"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + my $good = 0; + if ($success) { + my ($nchips) = split " ", join "", @$stdout_buf; + $nchips += 0; + if ($nchips and $nchips == 60) { + $good = 1; + } else { + print "$inst has less than 60 psf extensions: $nchips checking with censorObjects\n"; + my $command = "$censorObjects -checkinputonly -file $inst"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + if ($success) { + $good = 1; + } else { + print "$command failed with error_code: $error_code\n", + } + } + } else { + print "$command failed with error_code: $error_code\n", + } + if (!$good) { + push @{$file{bad_instances}}, $inst; + next; + } + } else { + my $command = "$checkfits $inst"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print "$command failed with error_code: $error_code\n", + push @{$file{bad_instances}}, $inst; + next; + } + } + } + if (!defined $good_inst) { + $good_inst = $inst; + $good_size = $this_size; + $good_md5 = $this_md5; + $good_censored = $this_censored; + + push @{$file{instances}}, $inst; + + # unless we're asked for a full all really care about is the censored status last if !$full_check; - } elsif ($this_size != $first_size) { - print STDERR "size of $inst $this_size is not equal to size of $first_inst: $first_size\n"; + } elsif ($this_size != $good_size) { + print "size of $inst $this_size is not equal to size of $good_inst: $good_size\n"; $file{inconsistent} = 1; - } elsif ($this_md5 ne $first_md5) { - print STDERR "md5 of $inst $this_md5 is not equal to size of $first_inst: $first_md5\n"; + push @{$file{bad_instances}}, $inst; + } elsif ($this_md5 ne $good_md5) { + print "md5 of $inst $this_md5 is not equal to size of $good_inst: $good_md5\n"; $file{inconsistent} = 1; - } - } - $file{num_instances} = $available_instances; - $file{censored} = $first_censored; - $file{size} = $first_size; - $file{md5} = $first_md5; - } + push @{$file{bad_instances}}, $inst; + } else { + # this matches the good instance + push @{$file{instances}}, $inst; + } + } + $file{num_instances} = scalar @{$file{instances}}; + $file{size} = $good_size; + $file{md5} = $good_md5; + if ($good_inst) { + $file{censored} = $good_censored; + } else { + # no good instances see if there was a bad one and check whether its url + # looks like a censored one + if (scalar @{$file{bad_instances}}) { + my $bad = ${$file{bad_instances}}->[0]; + $file{censored} = ($bad =~ /SR_/); + print "no good instances found setting censored based on $bad\n"; + } + } + } + } else { + print "No storage object found for $url \n"; } @@ -320,78 +435,62 @@ } +sub rename_file { + if ($no_op) { + return 1; + } + + my $original_url = shift; + my $backup_url = shift; + + # rename the backup as the orig + print "No storage object for original url: $original_url\n"; + print "Replacing with file at : $backup_url\n"; + + my $moved; + eval { + $moved = nebulous->move($backup_url, $original_url); + }; + if ($@ or not $moved) { + print "Failed to move $backup_url to $original_url\n"; + &my_die( "aborting", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + } +} + sub repair_file { + if ($no_op) { + return 1; + } + my $file = shift; my $smf = shift; - if ($file->{num_instances} < 2) { - print STDERR "cannot repair file with only one instance\n"; + + if ($file->{num_instances} < 1) { + print "cannot repair file with only no good instances\n"; return 0; } - my $good_inst; - my $good_size; - my $good_md5; - # try and find a "good instance" - my @good; - my $i; - for ($i = 0; $i < scalar @{$file->{instances}}; $i++; { - # assume the worst - $good[$i] = 0; - if (! -e $inst ) { - next; - } - my $size = -s $inst; - if (! $size ) { - next; - } - my $md5 = file_md5_hex($inst); - if ($good_inst) { - if (($size == $good_size) and ($md5 eq $good_md5)) { - $good[$i] = 1; - } - next; - } - - if (!$smf) { - my $command = "$checkfits $inst"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if ($success) { - if (!$good_inst) { - $good_inst = $inst; - $good_$md5 = $md5; - $good_size = $size; - } - $good[$i] = 1; - } - } else { - # Need to write this program - my $command = "$checksmf $inst"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if ($success) { - if (!$good_inst) { - $good_inst = $inst; - $good_$md5 = $md5; - $good_size = $size; - } - $good[$i] = 1; - } - } - } - - if (!$good_inst) { - print STDERR "unable to find any good instances of $file->{url}\n"; - return 0; - } - - my $num_good = 0; - for (my $j = 0; $j <= $i; $j++) { - $num_good++ if $good[$j]; - } - for (my $j = 0; $j <= @{$file->{bad_instances}}; $j++) { - $good[$i++] = 0; + + # file wouldn't get into instances array unless we decided that it was good in check_file + my $good_inst = $file->{instances}->[0]; + + my $fixed = 0; + foreach my $bad_inst (@{$file->{bad_instances}}) { + print "Attempting to repair $bad_inst\n"; + + my $parent_directory = dirname($bad_inst); + if (!-d $parent_directory) { + if (! mkdir $parent_directory) { + print "failed to mkdir parent directory $parent_directory\n"; + $fixed = 0; + last; + } + if (!copy $good_inst, $bad_inst) { + print "failed to copy $good_inst to $bad_inst\n"; + } + $fixed = 1; + push @{$file->{instances}}, $bad_inst; + } } - - return 0; + return $fixed; } @@ -422,10 +521,7 @@ my $msg = shift; # Warning message on die my $magic_ds_id = shift; # Magic DS identifier - my $component = shift; # class_id or skycell_id my $exit_code = shift; # Exit code to add carp($msg); - - $component = "exposure"; $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;