IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 37025


Ignore:
Timestamp:
Jul 16, 2014, 3:29:14 PM (12 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20140610/ippScripts
Files:
9 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20140610/ippScripts/Build.PL

    r36810 r37025  
    136136        scripts/psphot_fullforce_summary.pl
    137137        scripts/stack_bkg_mk_mdc.pl
     138        scripts/sc_mk_stack_mdc.pl
     139        scripts/sc_prepare_camera.pl
     140        scripts/sc_prepare_chip.pl
     141        scripts/sc_prepare_stack.pl
     142        scripts/sc_prepare_warp.pl
     143        scripts/sc_remote_exec.pl
     144        scripts/sc_transfer_tool.pl
     145        scripts/sc_validate_files.pl
     146        scripts/sc_validate_processing.pl
    138147    )],
    139148    dist_abstract => 'Scripts for running the Pan-STARRS IPP',

  • branches/eam_branches/ipp-20140610/ippScripts/MANIFEST

    r36704 r37025  
    4949scripts/lap_science.pl
    5050scripts/regenerate_background.pl
     51scripts/sc_mk_stack_mdc.pl
     52scripts/sc_prepare_camera.pl
     53scripts/sc_prepare_chip.pl
     54scripts/sc_prepare_stack.pl
     55scripts/sc_prepare_warp.pl
     56scripts/sc_remote_exec.pl
     57scripts/sc_transfer_tool.pl
     58scripts/sc_validate_files.pl
     59scripts/sc_validate_processing.pl
    5160t/00_distribution.t

  • branches/eam_branches/ipp-20140610/ippScripts/scripts/queuestaticsky.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_prepare_camera.pl

    r36884 r37025  
    351351   
    352352    if (defined $id and not $no_update) {
    353         my $command = "remotetool -updaterun -stage_id $id";
     353        my $command = "remotetool -updaterun -remote_id $id";
    354354        $command .= " -fault $exit_code " if defined $exit_code;
    355355        $command .= " -set_state $exit_state " if defined $exit_state;

  • branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_prepare_chip.pl

    r36884 r37025  
    127127        $detselect_command2   .= " -det_id $det_id ";
    128128        $detselect_command2   .= " -iteration $det_iter ";
     129        $detselect_command2   .= " -dbname $dbname " if defined($dbname);
    129130        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    130131            run(command => $detselect_command2, verbose => 0);
    … …  
    534535   
    535536    if (defined $id and not $no_update) {
    536         my $command = "remotetool -updaterun -stage_id $id";
     537        my $command = "remotetool -updaterun -remote_id $id";
    537538        $command .= " -fault $exit_code " if defined $exit_code;
    538539        $command .= " -set_state $exit_state " if defined $exit_state;

  • branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_prepare_stack.pl

    r36928 r37025  
    168168        my $remote_path_base = uri_local_to_remote($warp_path_base);
    169169        $warp_path_base_string .= " $remote_path_base ";
     170
     171        # Append file names to transfer lists so we can recover if warps are deleted.
     172        uri_to_outputs($warp_path_base . ".fits");
     173        uri_to_outputs($warp_path_base . ".wt.fits");
     174        uri_to_outputs($warp_path_base . ".mask.fits");
     175        uri_to_outputs($warp_path_base . ".cmf");
     176#       uri_to_outputs($warp_path_base . ".psf");
    170177    }
    171178    my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl $warp_path_base_string > ${remote_outroot}.in.mdc";
    … …  
    350357   
    351358    if (defined $id and not $no_update) {
    352         my $command = "remotetool -updaterun -stage_id $id";
     359        my $command = "remotetool -updaterun -remote_id $id";
    353360        $command .= " -fault $exit_code " if defined $exit_code;
    354361        $command .= " -set_state $exit_state " if defined $exit_state;

  • branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_prepare_warp.pl

    r36884 r37025  
    2727my $max_time      = 8;                      # longest allocation to ask for
    2828my $remote_hostname= "LANL/Mustang";         # Name of the remote node.
    29 my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","LOG.EXP","TRACE.EXP");
     29my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SAVE","LOG.EXP","TRACE.EXP");
    3030# Look for programs we need
    3131my $missing_tools;
    … …  
    118118    # This actually returns all the individual warp/skyfiles that comprise this run.  Because consistency.
    119119    my $command = "$warptool -towarped -warp_id $warp_id ";
     120    $command   .= " -dbname $dbname " if defined($dbname);
    120121    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    121122        run(command => $command, verbose => 0);
    … …  
    366367   
    367368    if (defined $id and not $no_update) {
    368         my $command = "remotetool -updaterun -stage_id $id";
     369        my $command = "remotetool -updaterun -remote_id $id";
    369370        $command .= " -fault $exit_code " if defined $exit_code;
    370371        $command .= " -set_state $exit_state " if defined $exit_state;

  • branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_transfer_tool.pl

    r36928 r37025  
    1212my $local_raw = '/scratch3/watersc1/raw/';
    1313my $local_tmp = '/scratch3/watersc1/tmp/';
    14 my $threads = 8;
     14my $threads = 10;
    1515my @hosts = ('ippc20.ipp.ifa.hawaii.edu','ippc24.ipp.ifa.hawaii.edu','ippc28.ipp.ifa.hawaii.edu',
    1616             'ippc21.ipp.ifa.hawaii.edu','ippc25.ipp.ifa.hawaii.edu','ippc29.ipp.ifa.hawaii.edu',
    1717             'ippc22.ipp.ifa.hawaii.edu','ippc26.ipp.ifa.hawaii.edu',
    18              'ippc23.ipp.ifa.hawaii.edu','ippc27.ipp.ifa.hawaii.edu');
     18             'ippc23.ipp.ifa.hawaii.edu','ippc27.ipp.ifa.hawaii.edu'    );
     19@hosts = (@hosts, @hosts, @hosts, @hosts, @hosts);
    1920my $input_file;
    2021my $verbose = 0;
    2122my $fetch = 0;
     23my $offset = 0;
    2224
    2325GetOptions(
    … …  
    2527    'input=s'     => \$input_file,
    2628    'fetch=s'     => \$fetch,
     29    'offset=s'    => \$offset,
    2730    'verbose'     => \$verbose,
    2831    ) or pod2usage( 2 );
    … …  
    4548my @filehandles;
    4649my $i;
     50my $line = 0;
    4751for ($i = 0; $i < $threads; $i++) {
    4852    open($filehandles[$i], ">${local_tmp}/${input_base}.${i}");
    … …  
    5155while(<I>) {
    5256    chomp;
    53     $i = int(rand($#filehandles));
     57    $line++;
     58    if ($line < $offset) { next; } # I think this is off-by-one in a safe direction
     59
     60    $i = int(rand($#filehandles + 1));
    5461
    5562    my $fline;
    … …  
    5966    if (($fetch)&&(!(-e $_))) {  # We are fetching, and do not already have this file.
    6067        print { $filehandles[$i] } $fline;
    61 #       $i++;
    6268    }
    6369    elsif (!($fetch)) { # We are pushing
    64 #       if (-e "${local_tmp}/$_") {
    65             print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
    66 #           $i++;
    67 #       }
    68 #       else { # But somehow couldn't find the file we expected
    69 #           print STDERR "Expected file ${local_tmp}/$_ not found!\n";
    70 #           die;
    71 #       }
     70        print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
    7271    }
    73 #    if ($i >= $threads) { $i = 0; }
    7472}
    7573close(I);
    … …  
    8482    $pids[$i] = fork();
    8583    if ($pids[$i] == 0) {
    86         my $host = $hosts[rand($#hosts)];
    87         my $code;
     84        my $host = $hosts[$i];
     85        my $code = 0;
    8886        if ($fetch) {
    8987            $code = fetch_task($host,"${input_base}.${i}", 0);
    … …  
    9896    waitpid($pids[$i],0);
    9997}
    100 # die;
    101 # if ($fetch) {
    102 #     system("parallel -j $threads  ln -sf ${local_tmp}/{2} {1} :::: $input_file :::: $fetch");
    103 # }
    10498
    10599for ($i = 0; $i < $threads; $i++) {
    … …  
    117111    my $error = shift;
    118112
    119     my $command = "tar cf - --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date?
     113    # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date?
     114    my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference";
    120115    print STDERR "$command\n";
    121 #    die;
     116
    122117    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    123118        run(command => $command, verbose => $verbose);
    124119    unless ($success) {
    125         $error_code = (($error_code >> 8) or 4);
     120
     121        print STDERR "ERROR:  $error_code\n";
     122#       my $std = join "\n", @{ $stdout_buf };
     123#       print "STDOUT: $std\n";
     124#       $std = join "\n", @{ $stderr_buf };
     125#       print "STDERR: $std\n";
     126
     127        foreach my $line (@{ $stderr_buf }) {  # This is a hack.  A messy ugly hack.
     128            if ($line =~ /No such file or directory/) {
     129                my $file = (split /\s+/, $line)[1];
     130                $file =~ s/:$//;
     131                if ($file !~ /nebulous/) { next; }
     132                print STDERR "Touching $file so tar can continue.\n";
     133                system("touch $file");
     134            }
     135        }
     136
    126137        warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
    127138        $error++;
    … …  
    142153
    143154    system("/usr/projects/cosmo/amd6100/bin/scp -o NoneSwitch=yes -o NoneEnabled=yes ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/");
    144     my $command = "/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar cf - --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference ";
     155    my $command = "/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar cf - --ignore-failed-read  --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference ";
    145156    print STDERR "$command\n";
    146157

  • branches/eam_branches/ipp-20140610/ippScripts/scripts/skycell_jpeg.pl

    r36360 r37025  
    275275                    'exp'   => "PPSTACK.UNCONV.EXP",
    276276                    'num'   => "PPSTACK.UNCONV.EXPNUM",
    277                     'bkg'   => "PPSTACK.OUTPUT.BKGMODEL"
     277#                   'bkg'   => "PPSTACK.OUTPUT.BKGMODEL"
    278278        );
    279279   
Note: See TracChangeset for help on using the changeset viewer.