IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 38081


Ignore:
Timestamp:
Apr 2, 2015, 3:08:12 PM (11 years ago)
Author:
mhuber
Message:

changes specifically for ippmd:uhhpc stack processing

Location:
tags/ipp-mdpv3-20150326
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • tags/ipp-mdpv3-20150326/ippScripts/scripts/sc_mk_stack_mdc.pl

    r37833 r38081  
    99
    1010my $remote_root = '/scratch3/watersc1/';
     11$remote_root = '/lus/scratch/mehuber7/';
    1112
    1213my ($compmap_file);

  • tags/ipp-mdpv3-20150326/ippScripts/scripts/sc_prepare_stack.pl

    r38010 r38081  
    2121# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
    2222my $remote_root     = '/scratch3/watersc1/';  # Far side destination base location
     23$remote_root = '/lus/scratch/mehuber7/';
     24
    2325my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
     26$remote_hostname = "UH/Cray";
    2427my $threads_req     = 4;                      # How many threads are we going to use?
    25 my $have_warps      = 0;                      # Do we have a copy of the warp if we don't find one there?
     28my $have_warps      = 1;                      # Do we have a copy of the warp if we don't find one there?
    2629
    2730my $fail_state = "prep_fail";
    … …  
    210213    $ppstack_command    .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF ";
    211214    $ppstack_command    .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL ";
    212     $ppstack_command    .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID -R PPSTACK.OUTPUT FITS.TYPE COMP_STACK ";
    213     $ppstack_command    .= " -R PPSTACK.OUTPUT.VARIANCE FITS.TYPE COMP_STACK  -R PPSTACK.UNCONV FITS.TYPE COMP_STACK ";
    214     $ppstack_command    .= " -R PPSTACK.UNCONV.VARIANCE FITS.TYPE COMP_STACK";
     215    $ppstack_command    .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID ";
     216    ## large stacks should not be compressed, this is bad hardcoding
     217    #-R PPSTACK.OUTPUT FITS.TYPE COMP_STACK ";
     218    #$ppstack_command    .= " -R PPSTACK.OUTPUT.VARIANCE FITS.TYPE COMP_STACK  -R PPSTACK.UNCONV FITS.TYPE COMP_STACK ";
     219    #$ppstack_command    .= " -R PPSTACK.UNCONV.VARIANCE FITS.TYPE COMP_STACK";
     220    $ppstack_command    .= " -F PPSTACK.OUTPUT PPSTACK.OUTPUT.NOCOMP -F PPSTACK.OUTPUT.VARIANCE PPSTACK.OUTPUT.VARIANCE.NOCOMP ";
     221    $ppstack_command    .= " -F PPSTACK.OUTPUT.EXPWT PPSTACK.OUTPUT.EXPWT.NOCOMP -F PPSTACK.OUTPUT.EXP PPSTACK.OUTPUT.EXP.NOCOMP ";
     222    $ppstack_command    .= " -F PPSTACK.UNCONV PPSTACK.UNCONV.NOCOMP -F PPSTACK.UNCONV.VARIANCE PPSTACK.UNCONV.VARIANCE.NOCOMP ";
     223    $ppstack_command    .= " -F PPSTACK.UNCONV.EXPWT PPSTACK.UNCONV.EXPWT.NOCOMP -F PPSTACK.UNCONV.EXP PPSTACK.UNCONV.EXP.NOCOMP ";
     224    ##
    215225    $ppstack_command    .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log ";
    216226    $ppstack_command    .= " -threads $threads_req ";

  • tags/ipp-mdpv3-20150326/ippScripts/scripts/sc_remote_exec.pl

    r37833 r38081  
    1616# Hard coded values
    1717my $DMZ_HOST = 'wtrw';
     18$DMZ_HOST = '128.171.123.18';
     19
    1820my @SEC_HOSTS= ('mu-fe1','mu-fe2','mu-fe3'
    1921                # ,'mu-fe4'
    2022    );
    2123my $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; # 'mu-fe';
     24$SEC_HOST = '';
     25
    2226my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
     27$IPP_PATH = '/home/mehuber7/src/';
     28
    2329my $remote_root  = '/scratch3/watersc1/';
     30$remote_root = '/lus/scratch/mehuber7/';
    2431
    2532# tools
    … …  
    111118# Run real command
    112119my (undef,$remote_command) = &uri_convert($uri_command);
    113 my $ssh_exec_stdout = &ssh_exec_command("msub -V $remote_command");
     120my $ssh_exec_stdout = &ssh_exec_command("sbatch $remote_command");   ##"msub -V $remote_command");
    114121if ($#{ $ssh_exec_stdout } != -1) { # Parse the output
    115     $job_id = ${ $ssh_exec_stdout }[0];
    116     chomp($job_id);
     122    my $line = ${ $ssh_exec_stdout }[0];
     123    chomp($line);
     124    my @line_split = split /\s+/, $line;
     125    $job_id = $line_split[-1];
    117126    $job_id =~ s/\s+//g;
    118127}
    … …  
    141150    my $file = shift;
    142151    my $destination = shift;
    143     my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}";
     152    my $cmd;
     153    if ($SEC_HOST ne '') {
     154        $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}";
     155    }
     156    else {
     157        $cmd = "$scp $file ${DMZ_HOST}:${destination}";
     158    }
    144159
    145160    my $directory = dirname($destination);
    … …  
    157172    my $destination = shift;
    158173    my $file = shift;
    159 
    160     my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file ";
     174    my $cmd;
     175    if ($SEC_HOST ne '') {
     176        $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}";
     177    }
     178    else {
     179        $cmd = "$scp $file ${DMZ_HOST}:${destination}";
     180    }
    161181
    162182    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    170190sub ssh_exec_command {
    171191    my $cmd = shift;
    172     $cmd = "$ssh -n $DMZ_HOST ssh  ${SEC_HOST} $cmd";
     192    if ($SEC_HOST ne '') {
     193        $cmd = "$ssh -n $DMZ_HOST ssh  ${SEC_HOST} $cmd";
     194    }
     195    else {
     196        $cmd = "$ssh -n $DMZ_HOST $cmd";
     197    }
     198
    173199    print "EXEC: $cmd\n";
    174200    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

  • tags/ipp-mdpv3-20150326/ippScripts/scripts/sc_remote_poll.pl

    r37833 r38081  
    2929# Hard coded values
    3030my $DMZ_HOST = 'wtrw';
     31$DMZ_HOST = '128.171.123.18';
    3132my @SEC_HOSTS= ('mu-fe1','mu-fe2','mu-fe3'
    32                 # ,'mu-fe4'
     33                ,'mu-fe4'
    3334    );
    3435my $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; # 'mu-fe';
     36$SEC_HOST = '';
    3537my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
     38$IPP_PATH = '/home/mehuber7/src/';
    3639my $remote_root  = '/scratch3/watersc1/';
     40$remote_root = '/lus/scratch/mehuber7/';
    3741
    3842# tools
    … …  
    351355    my $file = shift;
    352356    my $destination = shift;
    353     my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}";
     357    my $cmd;
     358    if ($SEC_HOST ne '') {
     359        $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}";
     360    }
     361    else {
     362        $cmd = "$scp $file ${DMZ_HOST}:${destination}";
     363    }
    354364
    355365    my $directory = dirname($destination);
    … …  
    368378    my $file = shift;
    369379
    370     my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file ";
     380    my $cmd;
     381    if ($SEC_HOST ne '') {
     382        $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file ";
     383    }
     384    else {
     385        $cmd = "$scp ${DMZ_HOST}:${destination} $file ";
     386    }
    371387
    372388    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    380396sub ssh_exec_command {
    381397    my $cmd = shift;
    382     $cmd = "$ssh -n $DMZ_HOST ssh  ${SEC_HOST} $cmd";
     398    if ($SEC_HOST ne '') {
     399        $cmd = "$ssh -n $DMZ_HOST ssh  ${SEC_HOST} $cmd";
     400    }
     401    else {
     402        $cmd = "$ssh -n $DMZ_HOST $cmd";
     403    }
    383404    print "EXEC: $cmd\n";
    384405    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

  • tags/ipp-mdpv3-20150326/ippScripts/scripts/sc_transfer_tool.pl

    r37833 r38081  
    1111
    1212my $scp_cmd = "/usr/projects/cosmo/amd6100/bin/scp";
     13$scp_cmd = '/usr/bin/scp';
    1314my $ssh_cmd = "/usr/projects/cosmo/amd6100/bin/ssh";
     15$ssh_cmd = '/usr/bin/ssh';
    1416
    1517my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/';
    16    $remote_root = '/scratch3/watersc1/';
     18#   $remote_root = '/scratch3/watersc1/';
     19$remote_root = '/lus/scratch/mehuber7/';
    1720
    1821#my $local_raw = "${remote_root}/tmp/";

  • tags/ipp-mdpv3-20150326/ippTasks/remote.pro

    r37835 r38081  
    2222
    2323list STAGES
    24   chip
    25   camera
     24#  chip
     25#  camera
    2626#  warp
    2727  stack
    28   staticsky
     28#  staticsky
    2929end
    3030
    … …  
    121121
    122122    # min entry limit?
    123     $run = remotetool -definebyquery -label $label -stage $stage -path_base neb://@HOST@.0/remote/$label -limit 500
     123    $run = remotetool -definebyquery -label $label -stage $stage -path_base neb://@HOST@.0/remote/$label -limit 200
    124124    if ($DB:n == 0)
    125125      option DEFAULT
    … …  
    156156  periods     -exec 900
    157157  periods     -timeout 30
    158   active      true
     158  active      false
    159159  npending    1
    160160
    … …  
    293293    end
    294294    if ("$STAGE" == "staticsky")
    295       $command = sc_prepare_staticsky.pl --camera GPC1 --sky_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot -dbname $DBNAME
     295      $command = sc_prepare_staticsky.pl --camera GPC1 --sky_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME
    296296    end
    297297
Note: See TracChangeset for help on using the changeset viewer.