IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 37282


Ignore:
Timestamp:
Aug 21, 2014, 4:22:03 PM (12 years ago)
Author:
eugene
Message:

update sc_prepare_run.pl

File:
1 edited

Legend:

Unmodified
Added
Removed

  • tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl

    r37278 r37282  
    2121use Pod::Usage qw( pod2usage );
    2222
    23 
    2423# Hard coded values
    25 # my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
    26 my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
    27 $remote_root = '/scratch3/watersc1/';
     24# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
     25my $remote_root = '/scratch3/watersc1/';
    2826my $remote_raw    = "${remote_root}/raw/";  # Directory to find raw data in.
     27
     28# value for chip:
    2929my $threads       = 2;                      # How many threads are we going to use?
    3030my $job_cost      = 150 / 60 / 60;          # Estimate of how long a job runs, in hours.
     31
    3132my $proc_per_node = 24;                     # processors per node
    3233my $min_nodes     = 1;                      # smallest allocation to ask for
    … …  
    4041# Look for programs we need
    4142my $missing_tools;
    42 my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
    4343my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
    4444my $detselect= can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1);
    … …  
    7171my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    7272
    73 #
    74 # Step 1: Get a list of the components that make up this remoteRun
    75 
    76 # SHould this call listrun to ensure we're in state new?
    77 my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id";
    78 $rt_cmd   .= " -dbname $dbname " if defined($dbname);
    79 
    80 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    81     run(command => $rt_cmd, verbose => $verbose);
    82 unless ($success) {
    83     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    84    
    85     &my_die("Unable to run chiptool to determine stage parameters.",
    86             $remote_id,$error_code);
    87 }
    88 my $compData = $mdcParser->parse(join "", @$stdout_buf) or
    89     &my_die("Unable to determine component information.",
    90             $remote_id,$PS_EXIT_PROG_ERROR);
    91 my $compData2= parse_md_list($compData);
    92 
    93 #
    94 # Step 1a: Load up the catalog of detrends
    95 # Detrend concept holders
    96 my %detrends = ();
    97 my %det_types = ('MASK' => '-mask',
    98                  'FLAT' => '-flat',
    99                  'DARK' => '-dark',
    100                  'VIDEODARK' => '-dark',
    101                  'LINEARITY' => '-linearity',
    102                  'FRINGE'    => '-fringe',
    103                  'NOISEMAP'  => '-noisemap');
    104 
    105 foreach my $det_type (keys (%det_types)) {
    106     my $dt_cmd = "$detselect -show -camera $camera -det_type $det_type ";
    107     $dt_cmd   .= " -dbname $dbname " if defined($dbname);
    108 
    109     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    110         run(command => $dt_cmd, verbose => $verbose);
     73# STEP 1: Get a list of the components that make up this remoteRun
     74my $compData;
     75{
     76    my $command = "$remotetool -listcomponent -remote_id $remote_id -state prep_done";
     77    $command   .= " -dbname $dbname " if defined($dbname);
     78
     79    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     80        run(command => $command, verbose => $verbose);
    11181    unless ($success) {
    11282        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    113         &my_die("UNable to run detselect to determine detrend catalog",
    114                 $remote_id,$error_code);
    115     }
    116     my $detData = $mdcParser->parse(join "", @$stdout_buf) or
    117         &my_die("Unable to determine detrend information.",
    118                 $remote_id,$PS_EXIT_PROG_ERROR);
    119     my $detData2= parse_md_list($detData);
    120 
    121     # Data structure for this:
    122     # $detrend{NAME} = \@list_of_detselect_shows
    123     # $detrend{NAME}[0] = \%hash_of_detselect_constraint_results (first element)
    124     # $detrend{NAME}[0]{class_id} = uri
    125 
    126     # At this point, I have the list of detselect shows, assign it
    127     $detrends{$det_type} = $detData2;
    128 
    129     for (my $dd = 0; $dd <= $#{ @{ $detrends{$det_type} } }; $dd++) {
    130         my $det_id = ${ $detrends{$det_type} }[$dd]->{det_id};
    131         my $det_iter=${ $detrends{$det_type} }[$dd]->{iteration};
    132 
    133         my $detselect_command2 = "detselect -select ";
    134         $detselect_command2   .= " -det_id $det_id ";
    135         $detselect_command2   .= " -iteration $det_iter ";
    136         $detselect_command2   .= " -dbname $dbname " if defined($dbname);
    137         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    138             run(command => $detselect_command2, verbose => 0);
    139         unless ($success) {
    140             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    141            
    142             &my_die("No valid detrend available for this image: $det_type $det_id $det_iter",
    143                     $chip_id,$error_code);
    144         }
    14583       
    146         my $detImfile = $mdcParser->parse(join "", @$stdout_buf) or
    147             &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter",
    148                     $chip_id,$error_code);
    149         my $detImfile2= parse_md_list($detImfile);
    150         foreach $detImfile (@$detImfile2) {
    151             my $class_id = $detImfile->{class_id};
    152             my $duri = $detImfile->{uri};
    153            
    154             ${ $detrends{$det_type} }[$dd]->{$class_id} = $duri;
    155         }
    156     }
    157 }
    158 
    159 #print Dumper(%detrends);
    160 
    161 my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN");
    162 
    163 #
    164 # Step 1b: Open output files
     84        &my_die("Unable to run chiptool to determine stage parameters.", $remote_id, $error_code);
     85    }
     86    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
     87        &my_die("Unable to determine component information.", $remote_id,$PS_EXIT_PROG_ERROR);
     88    $compData = parse_md_list($MDlist);
     89}
     90
     91# STEP 2: Open master output files
    16592my $uri_command = $path_base . ".cmd";
    16693my $uri_transfer= $path_base . ".transfer";
    … …  
    177104my $disk_return  = $ipprc->file_resolve($uri_return,1);
    178105
    179 my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it.
     106my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. 
    180107
    181108open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer",$chip_id,$PS_EXIT_SYS_ERROR);
    182 open(CHECK,    ">$disk_check")  || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR);
    183 open(CONFIG,   ">$disk_config")  || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR);
    184 open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR);
    185 open(RETURN,   ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR);
    186 
    187 
    188 #
     109open(CHECK,    ">$disk_check")     || &my_die("Couldn't open file? $disk_check",   $chip_id,$PS_EXIT_SYS_ERROR);
     110open(CONFIG,   ">$disk_config")    || &my_die("Couldn't open file? $disk_config",  $chip_id,$PS_EXIT_SYS_ERROR);
     111open(GENERATE, ">$disk_generate")  || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR);
     112open(RETURN,   ">$disk_return")    || &my_die("Couldn't open file? $disk_return",  $chip_id,$PS_EXIT_SYS_ERROR);
     113
    189114# Step 2: Iterate over all componenets in this remote run.
    190115my $job_index = 0;
    191 my @pre_commands = ();
    192 my @main_commands = ();
    193 my @post_commands = ();
    194 
    195 foreach my $compEntry (@$compData2) {
    196 
    197     my $chip_id = $compEntry->{stage_id};
    198     my $exp_id ;
    199 # Get exposure level information from the chipRun we're working from.
    200     my $command = "$chiptool -listrun -chip_id $chip_id ";
    201     $command   .= " -dbname $dbname " if defined($dbname);
    202    
    203     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    204         run(command => $command, verbose => 0);
    205     unless ($success) {
    206         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    207        
    208         &my_die("Unable to run chiptool to determine stage parameters.",
    209                 $chip_id,$error_code);
    210     }
    211    
    212 # Parse chipRun level data to determine detrend information
    213     my %detrends_to_use = ();
    214     {
    215         my $chipData = $mdcParser->parse(join "", @$stdout_buf) or
    216             &my_die("Unable to determine chip component information.",
    217                     $chip_id,$PS_EXIT_PROG_ERROR);
    218         my $chipData2= parse_md_list($chipData);
    219         my $chipEntry = ${ $chipData2 }[0];
    220        
    221         $exp_id = $chipEntry->{exp_id};   
    222         my $filter = $chipEntry->{filter};
    223         my $altfilt= $filter;
    224         $altfilt =~ s/.00000//;
    225         my $dateobs= $chipEntry->{dateobs};
    226         my ($date, $time) = split /T/, $dateobs;
    227         my ($year,$month,$day) = split /-/, $date;
    228         my ($hour,$minute,$second) = split /:/, $time; #/;
    229         my $dateobs_obj = DateTime->new(year => $year, month => $month, day => $day, hour => $hour, minute => $minute, second => $second);
    230 
    231         foreach my $det_type (%detrends) {
    232             if (($filter !~ /y/)&&($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y
    233 
    234             foreach my $det_obj (@{ $detrends{$det_type} }) { # Iterate over the available options
    235                 if (defined($det_obj->{filter})) {
    236                     if (($det_obj->{filter} ne $filter)&&($det_obj->{filter} ne $altfilt)) {
    237                         next; # Skip detrends with a filter that we can't use.
    238                     }
    239                 }
    240                 if (defined($det_obj->{time_begin})) {
    241                     if (DateTime->compare($dateobs_obj,$det_obj->{time_begin}) == -1) {
    242                         next; # Skip detrends that come into effect after this exposure (d1 < d2)
    243                     }
    244                 }
    245                 if (defined($det_obj->{time_end})) {
    246                     if (DateTime->compare($dateobs_obj,$det_obj->{time_end}) == 1) {
    247                         next; # Skip detrends that stop working before this exposure (d1 > d2)
    248                     }
    249                 }
    250                 $detrends_to_use{$det_type} = $det_obj;
    251                 last;
    252             }
    253         }
    254     }
    255 
    256 #
    257 # Step 3: Iterate over the sub-components
    258 # Iterate over the chipProcessedImfile level data.
    259     my $reg_command = "$chiptool -pendingimfile ";
    260     $reg_command   .= " -chip_id $chip_id " if defined($chip_id);
    261     $reg_command   .= " -dbname $dbname "   if defined($dbname);
    262    
    263     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    264         run(command => $reg_command, verbose => $verbose);
    265     unless ($success) {
    266         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    267        
    268         &my_die("Unable to run chiptool -pendingimfile",
    269                 $chip_id,$error_code);
    270     }
    271    
    272     my $chipData = $mdcParser->parse(join "", @$stdout_buf) or
    273         next;
    274 #       &my_die("Unable to parse chiptool -pendingimfile information.",
    275 #               $chip_id,$error_code);
    276     my $chipData2= parse_md_list($chipData);
    277 
    278    
    279     foreach my $chipEntry (@$chipData2) {
    280         # Get information we need to pass to ppImage
    281         my $uri            = $chipEntry->{uri};
    282         my $class_id       = $chipEntry->{class_id};
    283         my $video_cells    = $chipEntry->{video_cells};
    284         my $reduction      = $chipEntry->{reduction};
    285         my $exp_tag        = $chipEntry->{exp_tag};
    286         my $workdir        = $chipEntry->{workdir};
    287         my $chip_imfile_id = $chipEntry->{chip_imfile_id};
    288        
    289         # Process the image and burntool table
    290         my ($ipp_uri,$remote_uri) = uri_to_outputs_raw($uri);
    291         my $btt = $uri;
    292         $btt =~ s/fits$/burn.tbl/;
    293         my ($ipp_btt,$remote_btt) = uri_to_outputs_raw($btt);
    294        
    295         # Initialize the ppI command
    296         my $ppImage_command = "ppImage -file $remote_uri";
    297         $ppImage_command   .= " -burntool $remote_btt ";
    298        
    299         # Add detrend information to the command line
    300         foreach my $det (keys %detrends_to_use) {
    301             if ((($video_cells)&&($det eq 'DARK'))||
    302                 ((!$video_cells)&&($det eq 'VIDEODARK'))) {
    303                 next;
    304             }
    305             my $det_code = $det_types{$det};
    306             my $duri = $detrends_to_use{$det}->{$class_id};
    307             my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri);
    308             $ppImage_command .= " $det_code $remote_det_uri ";
    309         }
    310            
    311        
    312         # Add output root
    313         my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}";
    314         my $remote_outroot = uri_local_to_remote($ipp_outroot);
    315         $ppImage_command .= " $remote_outroot ";
    316         print STDERR "$remote_outroot $ipp_outroot $class_id\n";
    317         # Complete reduction information.
    318         $reduction = 'DEFAULT' unless defined $reduction;
    319         my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage
    320         my $recipe_psphot  = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot
    321        
    322         $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage ";
    323         $ppImage_command .= " -recipe PSPHOT $recipe_psphot ";
    324         $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats ";
    325         $ppImage_command .= " -threads 4 ";
    326         $ppImage_command .= " -image_id $chip_imfile_id ";
    327         $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace ";
    328         $ppImage_command .= " -log ${remote_outroot}.${class_id}.log ";
    329 
    330 #       push @main_commands, $ppImage_command;
    331 
    332         # Calculate pre and post commands
    333         my $remote_outroot_dir = dirname($remote_outroot);
    334         my $pre_command =  "mkdir -p $remote_outroot_dir";
    335        
    336         my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id ";
    337         $post_commandA   .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot ";
    338         $post_commandA   .= " -magicked 0 -hostname REMOTE -dtime_script 0 ";
    339         $post_commandA   .= " -dbname $dbname " if defined $dbname;
    340 
    341         my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo  ";
    342         my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo";
    343        
    344 #       push @post_commands, "$post_commandB $post_commandC";
    345 
    346         $job_index++;
    347 
    348         print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}";
    349 
    350 
    351         # Determine which output files need to be returned.
    352         foreach my $component (@return_component_list) {
    353 #           uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) );
    354             my $filename = $ipprc->filename($component, $ipp_outroot, $class_id);
    355             my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename );
    356             my $remote_outroot_dir = dirname($ipp_disk);
    357             print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
    358 #           print "  $filename $ipp_disk $remote_disk\n";
    359 #           die;
    360         }
    361 
    362         print CONFIG "\n";
    363     }
    364 }
    365 
    366 # Actually put the commands into the output file.
    367 # my %unique_cmds = ();
    368 # foreach my $cmd (@pre_commands) {
    369 #     unless(exists($unique_cmds{$cmd})) {
    370 #       print CONFIG $cmd . "\n";
    371 #       $unique_cmds{$cmd} = 1;
    372 #     }
    373 # }
    374 # foreach my $cmd (@main_commands) {
    375 #     unless(exists($unique_cmds{$cmd})) {
    376 #       print CONFIG $cmd . "\n";
    377 #       $unique_cmds{$cmd} = 1;
    378 #     }
    379 # }
    380 # foreach my $cmd (@post_commands) {
    381 #     unless(exists($unique_cmds{$cmd})) {
    382 #       print CONFIG $cmd . "\n";
    383 #       $unique_cmds{$cmd} = 1;
    384 #     }
    385 # }
     116foreach my $compEntry (@$compData) {
     117    my $line;
     118
     119    my $stage_id = $compEntry->{stage_id};
     120
     121    my $in_path_base = $compEntry->{path_base};
     122
     123    my $uri_transfer = $in_path_base . ".transfer";
     124    my $disk_transfer= $ipprc->file_resolve($uri_transfer);
     125    open(INPUT, "$disk_transfer");
     126    while ($line = <INPUT>) {
     127        print TRANSFER $line;
     128    }
     129    close (INPUT);
     130
     131    my $uri_check = $in_path_base . ".check";
     132    my $disk_check= $ipprc->file_resolve($uri_check);
     133    open(INPUT, "$disk_check");
     134    while ($line = <INPUT>) {
     135        print CHECK $line;
     136    }
     137    close (INPUT);
     138
     139    my $uri_config = $in_path_base . ".config";
     140    my $disk_config= $ipprc->file_resolve($uri_config);
     141    open(INPUT, "$disk_config");
     142    while ($line = <INPUT>) {
     143        print CONFIG $line;
     144    }
     145    close (INPUT);
     146
     147    my $uri_generate = $in_path_base . ".generate";
     148    my $disk_generate= $ipprc->file_resolve($uri_generate);
     149    open(INPUT, "$disk_generate");
     150    while ($line = <INPUT>) {
     151        print GENERATE $line;
     152    }
     153    close (INPUT);
     154
     155    my $uri_return = $in_path_base . ".return";
     156    my $disk_return= $ipprc->file_resolve($uri_return);
     157    open(INPUT, "$disk_return");
     158    while ($line = <INPUT>) {
     159        print RETURN $line;
     160    }
     161    close (INPUT);
     162
     163    $job_index += $compEntry->{jobs};
     164}
    386165
    387166close(CONFIG);
    … …  
    391170close(GENERATE);
    392171
    393 #
    394172# Construct the moab command last, so we can use the job_index counter to estimate resources.  Somehow.
    395173my $proc_need = $job_index * $threads;       # how many total processors do we need?
    … …  
    424202print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n";
    425203print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n";
    426 #print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n";
    427204print COMMAND "date\n";
    428205close(COMMAND);
    429 
    430206
    431207## We're done here. The execution and handling are done elsewhere.
    … …  
    441217}
    442218
    443 
    444    
    445 
    446 
     219exit (0);
    447220
    448221sub uri_convert {
    … …  
    462235}
    463236
    464 sub uri_convert_and_create {
    465     my $neb_uri = shift;
    466     my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1);
    467     my $remote_disk = $ipp_disk;
    468    
    469     unless(defined($ipp_disk)) {
    470         my_die();
    471     }
    472 
    473     $remote_disk =~ s%^.*/%%;   # Remove nebulous path
    474     $remote_disk =~ s%^\d+\.%%; # Remove ins_id
    475     $remote_disk =~ s%:%/%g;    # Replace colons with directories
    476     $remote_disk = "${remote_root}/${remote_disk}";
    477     return($ipp_disk,$remote_disk);
    478 }
    479 
    480 sub uri_to_outputs {
    481     my $neb_uri = shift;
    482     my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
    483    
    484     unless (exists($file_filter{$neb_uri})) {
    485         $file_filter{$neb_uri} = 1;
    486         print TRANSFER "$ipp_disk\n";
    487         print CHECK    "$remote_disk\n";
    488     }
    489     return($ipp_disk,$remote_disk);
    490 }
    491 
    492 sub uri_to_outputs_raw {
    493     my $neb_uri = shift;
    494     my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
    495     $remote_disk = $remote_raw . $ipp_disk;
    496     unless (exists($file_filter{$neb_uri})) {
    497         $file_filter{$neb_uri} = 1;
    498         print TRANSFER "$ipp_disk\n";
    499         print CHECK    "$remote_disk\n";
    500     }
    501     return($ipp_disk,$remote_disk);
    502 }
    503 
    504 sub uri_to_outputs_for_return {
    505     my $neb_uri = shift;
    506     my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri );
    507     unless (exists($file_filter{$neb_uri})) {
    508         $file_filter{$neb_uri} = 1;
    509         print RETURN "$ipp_disk\n";
    510         print GENERATE "$remote_disk\n";
    511     }
    512     return($ipp_disk,$remote_disk);
    513 }
    514 
    515 sub uri_local_to_remote {
    516     # This needs to replace the nebulous tag with the remote root.
    517     my $local_uri = shift;
    518     $local_uri =~ s%^.*?/%%; # neb:/
    519     $local_uri =~ s%^.*?/%%; # /
    520     $local_uri =~ s%^.*?/%%; # @HOST@.0/
    521     my $remote_uri = "${remote_root}/" . $local_uri;
    522 
    523     return($remote_uri);
    524 }
    525  
    526 sub uri_remote_to_local {
    527     # This needs to replace the remote root directory with the nebulous tag.
    528     my $remote_uri = shift;
    529     $remote_uri =~ s%${remote_root}%%;
    530     my $local_uri  = "neb:///" . $remote_uri;
    531    
    532     return($local_uri);
    533 }
    534237sub my_die {
    535238    my $msg = shift;
Note: See TracChangeset for help on using the changeset viewer.