IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 42450


Ignore:
Timestamp:
Apr 13, 2023, 2:17:26 PM (3 years ago)
Author:
eugene
Message:

better handle fail chips (bad md5sum)

Location:
trunk/tools/eam/rawfix.20230221/src
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl

    r42446 r42450  
    4646my $Ncul2 = 0;
    4747my $Ncul3 = 0;
     48my $Nfail = 0;
     49
     50my ($goodchips_r, $failchips_r) = &get_good_or_fail_chiplists ($topdir, $dateword, $stage);
    4851
    4952foreach my $chip_id (@chip_ids) {
    … …  
    5255    open ($outfh, ">$topdir/$dateword/$chip_id.uris.$stage.txt");
    5356
     57    print "open $topdir/$dateword/$chip_id.uris.$stage.txt\n";
     58
    5459    # get the list of chip neb paths for one chip
    5560    my ($neb_paths_r, $ds_array_r, $md_array_r) = &get_chip_paths ($fulldate, $chip_id);
    56 
    57     # I think this makes a copy of the arrays, which is inefficient
    58     # my @neb_paths = @$neb_paths_r;
    59     # my @ds_array  = @$ds_array_r;
    60     # my @md_array  = @$md_array_r;
    6161
    6262    # reset the hostcount for this chip
    … …  
    115115print "Ncul2: $Ncul2\n";
    116116print "Ncul3: $Ncul3\n";
    117 
    118 my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3;
     117print "Nfail: $Nfail\n";
     118
     119my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3 + $Nfail;
    119120if ($Ntotal != $Nchip) { die "ERROR: inconsistent count of chips\n"; }
    120121
    … …  
    132133$sql_update_stats .= ", Ncul2 = $Ncul2";
    133134$sql_update_stats .= ", Ncul3 = $Ncul3";
     135$sql_update_stats .= ", Nfail = $Nfail";
    134136$sql_update_stats .= " WHERE dateobs = $dateword";
    135137
    … …  
    293295    $Nchip ++;
    294296
     297    my $haveMD5 = 0;
     298    my $goodMD5 = 0;
     299    if ($goodchips_r->{$extID}) { $haveMD5 = 1; $goodMD5 = 1; }
     300    if ($failchips_r->{$extID}) { $haveMD5 = 1; $goodMD5 = 0; }
     301
     302    if ($extID eq "gpc1/20100503/o5319g0003d/o5319g0003d.ota46.fits") {
     303        print STDERR "$outinfo $Nmain $Nback $haveMD5 $goodMD5\n";
     304    }
     305
    295306    # GOOD : already fixed
    296307    if (($Nmain == 1) && ($Nback == 1)) {
    297308        print $outfh "GOOD: $outinfo\n";
    298         $Ngood ++;
     309        &update_counts (\$Ngood, \$Nfail, $haveMD5, $goodMD5);
    299310        return;
    300311    }
    … …  
    304315    if (($Nmain == 1) && ($Nback == 0)) {
    305316        print $outfh "SING: $outinfo\n";
    306         $Nsing ++;
     317        &update_counts (\$Nsing, \$Nfail, $haveMD5, $goodMD5);
    307318        return;
    308319    }
    … …  
    312323    if (($Nmain == 0) && ($Nback == 1)) {
    313324        print $outfh "SBCK: $outinfo\n";
    314         $Nsbck ++;
     325        &update_counts (\$Nsbck, \$Nfail, $haveMD5, $goodMD5);
    315326        return;
    316327    }
    … …  
    320331    if (($Nmain > 1) && ($Nback == 0)) {
    321332        print $outfh "DUPS: $outinfo\n";
    322         $Ndups ++;
     333        &update_counts (\$Ndups, \$Nfail, $haveMD5, $goodMD5);
    323334        return;
    324335    }
    … …  
    328339    if (($Nmain == 0) &&  ($Nback > 1)) {
    329340        print $outfh "XBCK: $outinfo\n";
    330         $Nxbck ++;
     341        &update_counts (\$Nxbck, \$Nfail, $haveMD5, $goodMD5);
    331342        return;
    332343    }
    … …  
    335346    if (($Nmain == 0) && ($Nback == 0) && $Nlost) {
    336347        print $outfh "DEAD: $outinfo\n";
    337         $Ndead ++;
     348        &update_counts (\$Ndead, \$Nfail, $haveMD5, $goodMD5);
    338349        return;
    339350    }
    … …  
    341352    if (($Nmain == 0) && ($Nback == 0) && ($Nlost == 0)) {
    342353        print $outfh "GONE: $outinfo\n";
    343         $Ngone ++;
     354        &update_counts (\$Ngone, \$Nfail, $haveMD5, $goodMD5);
    344355        return;
    345356    }
    … …  
    349360    if (($Nmain > 1) && ($Nback == 1)) {
    350361        print $outfh "CUL1: $outinfo\n";
    351         $Ncul1 ++;
     362        &update_counts (\$Ncul1, \$Nfail, $haveMD5, $goodMD5);
    352363        return;
    353364    }
    … …  
    357368    if (($Nback > 1) && ($Nmain == 1)) {
    358369        print $outfh "CUL2: $outinfo\n";
    359         $Ncul2 ++;
     370        &update_counts (\$Ncul2, \$Nfail, $haveMD5, $goodMD5);
    360371        return;
    361372    }
    … …  
    365376    if (($Nback > 1) && ($Nmain > 1)) {
    366377        print $outfh "CUL3: $outinfo\n";
    367         $Ncul3 ++;
     378        &update_counts (\$Ncul3, \$Nfail, $haveMD5, $goodMD5);
    368379        return;
    369380    }
    … …  
    372383    print $outfh "OOPS: $outinfo\n";
    373384    return;
     385}
     386
     387sub update_counts {
     388
     389    my $Nstat_r = $_[0];
     390    my $Nfail_r = $_[1];
     391    my $haveMD5 = $_[2];
     392    my $goodMD5 = $_[3];
     393
     394    ## if we know the MD5 is bad, add to Nfail, otherwise add to Ndups
     395    if ($haveMD5 && !$goodMD5) {
     396        $$Nfail_r ++;
     397    } else {
     398        $$Nstat_r ++;
     399    }
     400}
     401
     402sub get_good_or_fail_chiplists {
     403
     404    my $topdir   = $_[0];
     405    my $dateword = $_[1];
     406    my $stage    = $_[2];
     407
     408    # load the lists of good and fail chips
     409    my $prev = $stage - 1;
     410    my $goodfile = "$topdir/$dateword/goodchips.$prev.txt";
     411    my %goodchips;
     412    if (-e $goodfile) {
     413        open (GOODFILE,  $goodfile);
     414        my @lines = <GOODFILE>;
     415        close (GOODFILE);
     416       
     417        foreach my $line (@lines) {
     418            chomp $line;
     419            $goodchips{$line} = 1;
     420        }
     421    }
     422    my $failfile = "$topdir/$dateword/failchips.$prev.txt";
     423    my %failchips;
     424    if (-e $failfile) {
     425        open (FAILFILE,  $failfile);
     426        my @lines = <FAILFILE>;
     427        close (FAILFILE);
     428       
     429        foreach my $line (@lines) {
     430            chomp $line;
     431            $failchips{$line} = 1;
     432        }
     433    }
     434    return (\%goodchips, \%failchips);
    374435}
    375436

  • trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl

    r42449 r42450  
    2424my @vol_copy = &get_neb_volumes_copy; # array of neb hosts to copy data to at ITC
    2525my %vol_good = &get_neb_volumes_good; # hash of valid neb hosts
     26
     27my $Nchip = 0; # counts the number of instances
    2628
    2729my $Ngood = 0;
    … …  
    3840my $Nfail = 0;
    3941
    40 my $Nchip = 0; # counts the number of instances
    41 
    42 # load the lists of good and fail chips
    43 my $prev = $stage - 1;
    44 my $goodfile = "$topdir/$dateword/goodchips.$prev.txt";
    45 my %goodchips;
    46 if (-e $goodfile) {
    47     open (GOODFILE,  $goodfile);
    48     my @lines = <GOODFILE>;
    49     close (GOODFILE);
    50    
    51     foreach my $line (@lines) {
    52         chomp $line;
    53         $goodchips{$line} = 1;
    54     }
    55 }
    56 my $failfile = "$topdir/$dateword/failchips.$prev.txt";
    57 my %failchips;
    58 if (-e $failfile) {
    59     open (FAILFILE,  $failfile);
    60     my @lines = <FAILFILE>;
    61     close (FAILFILE);
    62    
    63     foreach my $line (@lines) {
    64         chomp $line;
    65         $failchips{$line} = 1;
    66     }
    67 }
     42my ($goodchips_r, $failchips_r) = &get_good_or_fail_chiplists ($topdir, $dateword, $stage);
    6843
    6944# read the list of chip_ids:
    … …  
    153128        my $haveMD5 = 0;
    154129        my $goodMD5 = 0;
    155         if ($goodchips{$ext_id}) { $haveMD5 = 1; $goodMD5 = 1; }
    156         if ($failchips{$ext_id}) { $haveMD5 = 1; }
     130        if ($goodchips_r->{$ext_id}) { $haveMD5 = 1; $goodMD5 = 1; }
     131        if ($failchips_r->{$ext_id}) { $haveMD5 = 1; $goodMD5 = 0; }
    157132
    158133        $Nchip ++;
    … …  
    350325}
    351326
     327sub get_good_or_fail_chiplists {
     328
     329    my $topdir   = $_[0];
     330    my $dateword = $_[1];
     331    my $stage    = $_[2];
     332
     333    # load the lists of good and fail chips
     334    my $prev = $stage - 1;
     335    my $goodfile = "$topdir/$dateword/goodchips.$prev.txt";
     336    my %goodchips;
     337    if (-e $goodfile) {
     338        open (GOODFILE,  $goodfile);
     339        my @lines = <GOODFILE>;
     340        close (GOODFILE);
     341       
     342        foreach my $line (@lines) {
     343            chomp $line;
     344            $goodchips{$line} = 1;
     345        }
     346    }
     347    my $failfile = "$topdir/$dateword/failchips.$prev.txt";
     348    my %failchips;
     349    if (-e $failfile) {
     350        open (FAILFILE,  $failfile);
     351        my @lines = <FAILFILE>;
     352        close (FAILFILE);
     353       
     354        foreach my $line (@lines) {
     355            chomp $line;
     356            $failchips{$line} = 1;
     357        }
     358    }
     359    return (\%goodchips, \%failchips);
     360}
     361
    352362# get the list of neb hosts to copy data to at ITC
    353363sub get_neb_volumes_copy {

  • trunk/tools/eam/rawfix.20230221/src/queue.rawfix.sh

    r42449 r42450  
    3131  set mynight = $2
    3232 
     33  set year = `echo $mynight | awk '{print substr($0, 1, 4)}'`
     34  if (($year < 2005) || ($year > 2030)) then
     35    echo "invalid date $mynight"
     36    exit 2
     37  endif
     38
     39  set month = `echo $mynight | awk '{print 1*substr($0, 5, 2)}'`
     40  if (($month < 1) || ($month > 12)) then
     41    echo "invalid date $mynight"
     42    exit 2
     43  endif
     44
     45  set day = `echo $mynight | awk '{print 1*substr($0, 7, 2)}'`
     46  if (($day < 1) || ($day > 31)) then
     47    echo "invalid date $mynight"
     48    exit 2
     49  endif
     50
    3351  mysql -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME -e "insert into nights (dateobs, state) values ('$mynight', 'new')"
    3452  exit 0

  • trunk/tools/eam/rawfix.20230221/src/rawfix.advance.pt

    r42446 r42450  
    11
    2 $MAX_ACTIVE_NIGHTS = 2
     2$MAX_ACTIVE_NIGHTS = 5
    33
    44## regularly run the command advance.rawfits.sh
Note: See TracChangeset for help on using the changeset viewer.