IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Mar 8, 2009, 4:50:46 PM (17 years ago)
Author:
eugene
Message:

merging changes from branches/eam_branches/eam_branch_20090303 (detrend normalization resequence, cleanup error state changes; SVN_VERSION from psbuild)

Location:
trunk
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk

  • trunk/ippScripts/scripts/detrend_norm_apply.pl

    r23186 r23229  
    3838    'iteration|n=s'     => \$iter,       # Iteration
    3939    'class_id|i=s'      => \$class_id,   # Class ID
    40     'value|v=s'         => \$value,      # Value to multiple (for normalisation)
     40    'value|v=s'         => \$value,      # Value to apply (for normalisation)
    4141    'input_uri|u=s'     => \$input_uri,  # Input file
    4242    'camera|c=s'        => \$camera,     # Camera

  • trunk/ippScripts/scripts/detrend_norm_calc.pl

    r23186 r23229  
    9090my @files;                      # The input files
    9191{
    92     my $command = "$dettool -processedimfile";
    93     $command .= " -det_id $det_id"; # Command to run
     92    my $command = "$dettool -residimfile";
     93    $command .= " -det_id $det_id";
     94    $command .= " -iteration $iter";
    9495    $command .= " -included"; # only use the inputs for this detrend run to calculate the norm
    9596    $command .= " -dbname $dbname" if defined $dbname;
    … …  
    9899    print "Running [$command]...\n" if $verbose;
    99100    if (not run(\@command, \$stdin, \$stdout, \$stderr)) {
    100         &my_die("Unable to perform dettool -processedimfile on detrend $det_id/$iter: $?",
     101        &my_die("Unable to perform dettool -residimfile on detrend $det_id/$iter: $?",
    101102                $det_id, $iter, $PS_EXIT_SYS_ERROR);
    102103    }

  • trunk/ippScripts/scripts/detrend_resid_exp.pl

    r23186 r23229  
    8888&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe;
    8989
     90# variables used for I/O
     91my ($command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
     92
     93# Get list of normalizations by class_id : stored as $norms; save to temp file for ppImage runs below
     94my (%norms, $normsName);
     95{
     96    # dettool command to select imfile data for this exp_id
     97    $command  = "$dettool -normalizedstat";
     98    $command .= " -det_id $det_id";
     99    $command .= " -iteration $iter";
     100    $command .= " -dbname $dbname" if defined $dbname;
     101    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     102        run(command => $command, verbose => $verbose);
     103    unless ($success) {
     104        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     105        warn("Unable to perform dettool -residimfile: $error_code\n");
     106        exit($error_code);
     107    }
     108    if (@$stdout_buf == 0) {
     109        &my_die("No normalizations were found", $det_id, $iter, $PS_EXIT_PROG_ERROR);
     110    }
     111
     112    # Parse the stdout buffer into a metadata
     113    my $mdcParser = PS::IPP::Metadata::Config->new;
     114    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     115        &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
     116
     117    # parse the file info in the metadata
     118    my $normsMD = parse_md_list($metadata) or
     119        &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
     120
     121
     122    # write the normalizations to a file as a metadata config file in the form: class_id F32 value
     123    # XXX a possible optimization: if there is only one imfile, skip normalization
     124    my $normsFile;
     125    ($normsFile, $normsName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.norms.XXXX", UNLINK => !$save_temps );
     126    print "saving norms to $normsName\n";
     127    foreach my $norm (@$normsMD) {
     128        my $class_id = $norm->{class_id};
     129        my $normalization = $norm->{norm};
     130
     131        $norms{$class_id} = $normalization;
     132        printf $normsFile "$class_id F32 $normalization\n",
     133    }
     134    close $normsFile;
     135}
     136
    90137# Get list of imfile files
    91138my $cmdflags;
    92 my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
     139my (@files);
    93140{
    94141    # dettool command to select imfile data for this exp_id
    … …  
    105152        exit($error_code);
    106153    }
    107     # XXX report an error message if stdout_buf is empty
     154    if (@$stdout_buf == 0) {
     155        &my_die("No imfiles were found", $det_id, $iter, $PS_EXIT_PROG_ERROR);
     156    }
    108157
    109158    # Parse the stdout buffer into a metadata
    … …  
    112161        &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
    113162
     163    # since I can't figure out how to do input and output within PERL, I'm writing the (modified) metadata to a temp file
     164    my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.stats.XXXX", UNLINK => !$save_temps );
     165
    114166    # parse the file info in the metadata
    115     $files = parse_md_list($metadata) or
    116         &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
    117 
    118     # since I can't figure out how to do input and output within PERL, I'm writing to a temp file
    119     my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.stats.XXXX", UNLINK => !$save_temps );
    120     print "saving stats to $statName\n";
    121     foreach my $line (@$stdout_buf) {
    122         print $statFile $line;
     167    # as we parse the list of files and their stats, apply the normalization to the relevant fields
     168    # also, write out the modified metadata set
     169    foreach my $mdItem (@$metadata) {
     170        if ($mdItem->{class} ne "metadata") {
     171            carp "MD element ", $mdItem->{name}, " isn't of type METADATA --- ignored.\n";
     172            next;
     173        }
     174        my %hash;               # Hash element
     175        my $mdComponents = $mdItem->{value}; # Components of the metadata
     176
     177        # determine the class_id for this block:
     178        my $class_id;
     179        foreach my $data (@$mdComponents) {
     180            unless ($data->{name} eq "class_id") { next; }
     181            $class_id = $data->{value};
     182            last;
     183        }
     184
     185        # a new metadata block
     186        print $statFile "rawResidImfile  METADATA\n";
     187
     188        # modify and save the data in this block:
     189        foreach my $data (@$mdComponents) {
     190            my $norm = $norms{$class_id};
     191
     192            # fields to modify by the normalization:
     193            if ($data->{name} eq "bg")            { $data->{value} *= $norm; }
     194            if ($data->{name} eq "bg_stdev")      { $data->{value} *= $norm; }
     195            if ($data->{name} eq "bg_mean_stdev") { $data->{value} *= $norm; }
     196            if ($data->{name} eq "bg_skewness")   { $data->{value} *= $norm; }
     197            if ($data->{name} eq "bg_kurtosis")   { $data->{value} *= $norm; }
     198            if ($data->{name} eq "bin_stdev")     { $data->{value} *= $norm; }
     199
     200            # write out the metadata, save on the array of hashes
     201            print $statFile "  $data->{name}  $data->{type}  $data->{value}\n";
     202            $hash{$data->{name}} = $data->{value};
     203        }
     204        print $statFile "END\n";
     205        push @files, \%hash;
    123206    }
    124207    close $statFile;
    … …  
    154237my ($list1File, $list1Name) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.b1.list.XXXX", UNLINK => !$save_temps );
    155238my ($list2File, $list2Name) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.b2.list.XXXX", UNLINK => !$save_temps );
    156 foreach my $file (@$files) {
     239foreach my $file (@files) {
    157240    print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n");
    158241    print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n");
    … …  
    165248unless ($no_op) {
    166249    # Make the jpeg for binning 1
     250    # XXX EAM : supply the collection of normalizations as a metadata
    167251    $command = "$ppImage -list $list1Name $outroot"; # Command to run
    168252    $command .= " -recipe PPIMAGE PPIMAGE_J1";
    169253    $command .= " -recipe JPEG $recipe";
     254    $command .= " -normlist $normsName";
    170255    $command .= " -dbname $dbname" if defined $dbname;
    171256    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    178263
    179264    # Make the jpeg for binning 2
     265    # XXX EAM : supply the collection of normalizations as a metadata
    180266    $command = "$ppImage -list $list2Name $outroot"; # Command to run
    181267    $command .= " -recipe PPIMAGE PPIMAGE_J2";
    182268    $command .= " -recipe JPEG $recipe";
     269    $command .= " -normlist $normsName";
    183270    $command .= " -dbname $dbname" if defined $dbname;
    184271    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    215302my @fluxes;
    216303
    217 foreach my $file (@$files) {
     304foreach my $file (@files) {
    218305    my $name      = $file->{class_id};
    219306    my $mean      = $file->{bg};        # Mean for this imfile

  • trunk/ippScripts/scripts/ipp_cleanup.pl

    r23186 r23229  
    4545my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up", $stage_id, $PS_EXIT_CONFIG_ERROR); # this is used for PATH, NEB filename conversions
    4646
    47 # $mode must be one of "goto_cleaned", "goto_scrubbed", or "goto_purged"
    48 # goto_cleaned and goto_scrubbed both result in 'cleaned': scrubbed allows chips without config files to
    49 # be cleaned (they cannot be recovered, but the small data is left behind)
     47# $mode must be one of "goto_cleaned", "goto_scrubbed", or
     48# "goto_purged" goto_cleaned and goto_scrubbed both result in
     49# 'cleaned' on success ('scrubbed' allows chips without config files
     50# to be cleaned; they cannot be recovered, but the small data is left
     51# behind). XXX make 'scrubbed' a data_state?
    5052unless (($mode eq "goto_cleaned") || ($mode eq "goto_scrubbed") || ($mode eq "goto_purged")) {
    5153    die "invalid cleanup mode $mode\n";
    5254}
     55
     56my $error_state;
     57if ($mode eq "goto_cleaned")  { $error_state = "error_cleaned";  }
     58if ($mode eq "goto_scrubbed") { $error_state = "error_scrubbed"; }
     59if ($mode eq "goto_purged")   { $error_state = "error_purged";   }
     60
    5361
    5462my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff  => 1);
    … …  
    148156            }
    149157        } else {
    150             # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will stop be run
    151             my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -code 1";
    152             $command .= " -dbname $dbname" if defined $dbname;
     158
     159            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
     160            my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -set_state $error_state";
     161            $command .= " -dbname $dbname" if defined $dbname;
    153162
    154163            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    216225
    217226    if ($status)  {
    218         my $command = "$camtool -cam_id $stage_id -updaterun";
     227        my $command;
    219228        if ($mode eq "goto_cleaned") {
    220             $command .= " -state cleaned";
    221         } else {
    222             $command .= " -state purged";
     229            $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";
     230        }
     231        if ($mode eq "goto_scrubbed") {
     232            $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";
     233        }
     234        if ($mode eq "goto_purged") {
     235            $command = "$camtool -updaterun -cam_id $stage_id -set_state purged";
    223236        }
    224237        $command .= " -dbname $dbname" if defined $dbname;
    … …  
    230243        }
    231244    } else {
    232         my $command = "$camtool -updateprocessedexp -cam_id $stage_id -code 1";
     245        # since 'camera' has only a single imfile, we can just update the run
     246        my $command = "$camtool -updaterun -cam_id $stage_id -set_state $error_state";
    233247        $command .= " -dbname $dbname" if defined $dbname;
    234248
    … …  
    319333            }
    320334         } else {
    321             # XXX: -updateskyfile mode does not exist, need to add it
    322             my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -code 1";
    323             $command .= " -dbname $dbname" if defined $dbname;
     335            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
     336            $command .= " -dbname $dbname" if defined $dbname;
    324337
    325338            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    337350# left TODO
    338351# fake : faketool : -pendingcleanupimfile (loop over imfiles)
    339 # stack: stacktool : -pendingcleanupskyfile (loop over skyfiles)
    340 # diff:  difftool : -pendingcleanupskyfile
     352# stack: stacktool : -pendingcleanupskyfile
     353# diff:  difftool : -pendingcleanupskyfile (loop over skyfiles)
    341354
    342355die "ipp_cleanup.pl -stage $stage not yet implemented\n";
    … …  
    367380}
    368381
     382# XXX we currently do not set the error state in the db on my_die
    369383sub my_die
    370384{
Note: See TracChangeset for help on using the changeset viewer.