IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 32992


Ignore:
Timestamp:
Dec 21, 2011, 11:57:23 AM (15 years ago)
Author:
heather
Message:

more merging

Location:
branches/haf_add201112/ippScripts/scripts
Files:
1 added
1 edited

Legend:

Unmodified
Added
Removed

  • branches/haf_add201112/ippScripts/scripts/minidvodb_premerge.pl

    r32989 r32992  
    1515my $dtime_resort;
    1616my $dtime_relphot;
    17 my $dtime_merge;
    18 my $dtime_verify;
    1917my $dtime_script;
    2018
    … …  
    4644}
    4745
    48 my ( $mergedvodb, $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps);
     46my ( $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps);
    4947GetOptions(
    50     'mergedvodb|w=s'         => \$mergedvodb,  # output DVO database
    5148    'minidvodb|w=s'     => \$minidvodb, #minidvodb database
    5249    'minidvodb_id|w=s'  => \$minidvodb_id, #minidvodb_id
    53  'minidvodb_group|w=s'  => \$minidvodb_group, #minidvodb_id
     50    'minidvodb_group|w=s'  => \$minidvodb_group, #minidvodb_id
    5451    'camera|c=s'        => \$camera, # Camera
    5552    'dbname|d=s'        => \$dbname, # Database name
    … …  
    6259pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    6360pod2usage(
    64           -msg => "Required options: --mergedvodb --minidvodb --minidvodb_id -- minidvodb_group --camera",
     61          -msg => "Required options: --minidvodb --minidvodb_id -- minidvodb_group --camera",
    6562          -exitval => 3,
    6663          ) unless
    67     defined $mergedvodb and
    6864    defined $minidvodb and
    6965    defined $minidvodb_id and
    … …  
    7167    defined $camera;
    7268
    73 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $mergedvodb, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
     69my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $minidvodb_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    7470
    7571if ($logfile) {
    76     $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $mergedvodb, $PS_EXIT_SYS_ERROR );
     72    $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $minidvodb_id, $PS_EXIT_SYS_ERROR );
    7773    print "\n\n";
    7874    print "Starting script $0 on $host\n\n";
    … …  
    8076}
    8177
    82 # convert supplied DVO database name to UNIX filename
    83 my $mergedvodbReal;
    84 if (defined $mergedvodb) {
    85     $mergedvodbReal = $ipprc->dvo_catdir( $mergedvodb ); # catdir for DVO
    86     $mergedvodbReal = $ipprc->convert_filename_absolute( $mergedvodbReal );
    87 }
    88 
    8978my $dtime_addstar = 0;
    90 #my $dtime_relphot = 0;
    9179
    9280unless ($no_op) {
    93     if (defined $mergedvodbReal) {
    94 
    95         #this is chopped into several parts: firstcheck, addstar, relphot, dvoverify, merge
    96        
    97         #first check that there were no faulted merges before. If there are, fault this merge with PS_EXIT_UNKNOWN_ERROR
    98         {
    99             my $nothing_faulted = 0;
    100             my $mdcParser = PS::IPP::Metadata::Config->new;
    101 
    102             my $command = "$addtool -listminidvodbprocessed -faulted -minidvodb_group " . $minidvodb_group;
    103             $command .= " -dbname $dbname" if defined $dbname;
    104 
    105             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    106                     run(command => $command, verbose => $verbose);
    107             &my_die( "Unable to get list of faulted minidvodbs", $minidvodb_id, $PS_EXIT_SYS_ERROR) unless $success;
    108             if (scalar @$stdout_buf == 0 ) { #it lists nothing if nothign has faulted
    109                 $nothing_faulted =1;
    110                 print "previous merges are okay.\n";
    111             }
    112             &my_die( "Previous merges faulted, do not proceed until they are investigated:", $minidvodb_group, $PS_EXIT_UNKNOWN_ERROR) unless $nothing_faulted;
    113 
    114 
    115  
    116         }
    117 
     81    {
     82        #this is chopped into several parts: addstar, relphot
    11883        #addstar
    11984        {
    … …  
    12186            $command .= " -D CAMERA $camera";
    12287            $command .= " -D CATDIR $minidvodb";
    123 
    12488            my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
    12589            print "\n$command\n";
    … …  
    12892            unless ($success) {
    12993                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    130                 &my_die("Unable to perform addstar: $error_code", $mergedvodb, $error_code);
     94                &my_die("Unable to perform addstar: $error_code", $minidvodb_id, $error_code);
    13195            }
    13296            $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);  $dtime_resort = $dtime_addstar;
    … …  
    151115            unless ($success) {
    152116                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    153                 &my_die("Unable to perform relphot: $error_code", $mergedvodb, $error_code);
     117                &my_die("Unable to perform relphot: $error_code", $minidvodb_id, $error_code);
    154118            }
    155119            $dtime_relphot = 86400.0*(DateTime->now->mjd - $mjd_relphot_start);   # MJD of starting script
    156120            print "relphot time $dtime_relphot\n";
    157121        }
    158         #dvoverify
     122
    159123        {
    160             my $command  = "$dvoverify -s $minidvodb";
    161             print "$command\n";
    162             my $mjd_dvoverify_start = DateTime->now->mjd;   # MJD of starting script
    163124
     125            $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start);
     126            my $command = "addtool -minidvodb_id $minidvodb_id";
     127            $command .= " -addminidvodbprocessed";
     128            $command .= " -minidvodb_group $minidvodb_group";
     129            $command .= " -dtime_relphot $dtime_relphot"  if defined $dtime_relphot;
     130            $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort;
     131            $command .= " -dtime_script $dtime_script" if defined $dtime_script;
     132            $command .= " -dbname $dbname" if defined $dbname;
     133            #print $command;
    164134            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    165135                run(command => $command, verbose => $verbose);
    166136            unless ($success) {
    167137                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    168                 &my_die("Unable to perform dvoverify: $error_code", $mergedvodb, $error_code);
    169             }
    170             $dtime_verify = 86400.0*(DateTime->now->mjd - $mjd_dvoverify_start);   # MJD of starting script
    171             print "dvoverify time $dtime_verify\n";
    172         }
    173 
    174 
    175         #Merge
    176 
    177         my $this_is_the_first;
    178         {
    179 
    180             my $mdcParser = PS::IPP::Metadata::Config->new;
    181 
    182             my $command = "$addtool -listminidvodbrun -state merged -minidvodb_group " . $minidvodb_group;
    183             $command .= " -dbname $dbname" if defined $dbname;
    184 
    185             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    186                     run(command => $command, verbose => $verbose);
    187             &my_die( "Unable to get list of minidvodbruns", $minidvodb_id, $PS_EXIT_SYS_ERROR) unless $success;
    188             if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first
    189                 $this_is_the_first =1;
    190                 print "listing nothing\n";
    191             } else {
    192                 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    193                     &my_die("Unable to parse metadata config", $minidvodb_id, $PS_EXIT_PROG_ERROR);
    194                 #this fails if there is nothing listed. I checked.
    195                 my $components = parse_md_list($metadata) or
    196                     &my_die("Unable to parse metadata list", $minidvodb_id, $PS_EXIT_PROG_ERROR);
    197                 my $comp = $$components[0];
    198                 my  $minidvodb_name = $comp->{minidvodb_name};
    199 
    200                 if (!defined($minidvodb_name)) {
    201                     &my_die("Unable to parse minidvodb_name", $minidvodb_id, $PS_EXIT_PROG_ERROR);
    202                 } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk.
    203                 print "found at least 1 minidvodbrun in merged state\n";
    204                 $this_is_the_first = 0;
     138                &my_die("Unable to add to minidvodbprocessed: $error_code", $minidvodb_id, $error_code);
    205139            }
    206140        }
    207         print "$this_is_the_first $mergedvodb/Image.dat\n";
    208        
    209          
    210         if (-e "$mergedvodb/Image.dat") {
    211             if ($this_is_the_first == 1) {
    212                 &my_die("refusing to merge, this is the first, but files already exist in dir", $minidvodb_id, 4);
    213             }
    214             $this_is_the_first =0;
    215         }
    216        
    217         print "$this_is_the_first $mergedvodb/Image.dat\n";
    218         {
    219             my $merge_command;
    220             my $mjd_merge_start = DateTime->now->mjd;   # MJD of starting script
    221             if ($this_is_the_first) {
    222                
    223                 $merge_command = "rsync -rvat $minidvodb/* $mergedvodb";
    224             } else {
    225                 $merge_command = "$dvomerge $minidvodb into $mergedvodb";
    226             }
    227            
    228             print "\n$merge_command\n";
    229             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    230                 run(command => $merge_command, verbose => $verbose);
    231             unless ($success) {
    232                     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    233                     &my_die("Unable to merge: $error_code", $mergedvodb, $error_code);
    234             }
    235                
    236             $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start);   # MJD of starting script
    237             print "merge time $dtime_merge\n";
    238 #       }
    239         }
    240 
    241         {
    242 
    243             $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start);
    244 
    245 
    246             my $command = "addtool -minidvodb_id $minidvodb_id";
    247             $command .= " -addminidvodbprocessed";
    248             $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal;
    249             $command .= " -minidvodb_group $minidvodb_group";
    250             $command .= " -dtime_relphot $dtime_relphot"  if defined $dtime_relphot;
    251             $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort;
    252             $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge;
    253             $command .= " -dtime_script $dtime_script" if defined $dtime_script;
    254             $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify;
    255             $command .= " -dbname $dbname" if defined $dbname;
    256             #print $command;
    257 
    258             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    259                 run(command => $command, verbose => $verbose);
    260             unless ($success) {
    261                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    262                 &my_die("Unable to add to minidvodbprocessed: $error_code", $mergedvodb, $error_code);
    263             }
    264         }
    265 
    266141    } else {
    267         &my_die("dvodb: $mergedvodb not found", $mergedvodb, $PS_EXIT_UNKNOWN_ERROR);
     142        &my_die("dvodb: minidvodb_id = $minidvodb_id not found", $minidvodb_id, $PS_EXIT_UNKNOWN_ERROR);
    268143    }
    269144} else {
    270     print "skipping processing for $mergedvodbReal\n";
     145    print "skipping processing for minidvodb_id = $minidvodb_id\n";
    271146}
    272147
    … …  
    279154    my $minidbvodb_id = shift;
    280155    my $exit_code = shift; # Exit code to add
    281 
    282     print STDERR "$msg $mergedvodb\n";
     156    print STDERR "$msg $minidvodb_id\n";
    283157
    284158if (defined $minidvodb_id ) {
    … …  
    286160    my $command = "addtool -minidvodb_id $minidvodb_id";
    287161    $command .= " -addminidvodbprocessed";
    288         $command .= " -fault $exit_code";
    289         $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal;
    290         $command .= " -minidvodb_group $minidvodb_group";
    291         $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot;
    292         $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort;
    293         $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge;
     162    $command .= " -fault $exit_code";
     163    $command .= " -minidvodb_group $minidvodb_group";
     164    $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot;
     165    $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort;
    294166    $command .= " -dtime_script $dtime_script" if defined $dtime_script;
    295     $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify;
    296         $command .= " -dbname $dbname" if defined $dbname;
    297 
    298 
    299 
    300     #print $command;
     167    $command .= " -dbname $dbname" if defined $dbname;
    301168    system ($command);
    302169    }
    303 
    304 
    305 
    306 
    307170    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
    308 
    309171    exit $exit_code;
    310172}
Note: See TracChangeset for help on using the changeset viewer.