IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jun 21, 2007, 2:10:11 PM (19 years ago)
Author:
Paul Price
Message:

Changes to multiple packages in order to use "reduction classes" to
specify recipe names. The idea is to give a group of recipes a
symbolic name, which distinguish how the reduction is to be performed,
without going to all the trouble of changing the recipes themselves.

For example, we might want to turn off the usual
overscan/bias/dark/shutter/flat processing and only do photometry if
we have pre-processed data. Gene also wants a similar ability for
building the flat-field correction.

Added a REDUCTION metadata to the camera configuration that contains
multiple METADATA entries that have recipes for different processing
steps. PS::IPP::Config has a 'reduction' function to return the
appropriate recipe, and this is now used in the scripts. Updated
ippTasks to use these where appropriate.

This necessitated changing the database, so that the desired reduction
class can be specified in the right places. Instructions on converting
an extant database are in dbconfig/changes.txt.

Tested with the SIMTEST, and it works.

There is currently no way of specifying the reduction class when an
exposure proceeds from 'register' to 'chip' (it has to be hacked in
the database directly: "update chipPendingExp set reduction =
'SOME_CLASS' where ..."), but it can be specified when adding a
detrend run or queuing chips with 'chiptool -queue' (the '-reduction'
option). Also, the 'recipe' columns remain throughout the detrend
tables, and they continue to be populated with the actual recipe used
(rather than the reduction class; but that is contained in the
detRun).

Location:
trunk/ippScripts/scripts
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/chip_imfile.pl

    r13926 r13937  
    2929use Pod::Usage qw( pod2usage );
    3030
    31 my $RECIPE = 'PPIMAGE_OBDSFRA'; # Recipe to use
    32 
    3331# Parse the command-line arguments
    3432my ($exp_id,                    # Exposure identifier
    … …  
    3634    $class_id,                  # Class identifier
    3735    $input,                     # Input FITS file
    38     $recipe,                    # Recipe to use
    3936    $camera,                    # Camera
    4037    $dbname,                    # Database name
    4138    $workdir,                   # Working directory, for output files
     39    $reduction,                 # Reduction class
    4240    $no_update,                 # Don't update the database?
    4341    $no_op,                     # Don't do any operations?
    … …  
    4846           'class_id=s'    => \$class_id,
    4947           'uri|u=s'       => \$input,
    50            'recipe=s'      => \$recipe,
    5148           'camera|c=s'    => \$camera,
    5249           'dbname|d=s'    => \$dbname, # Database name
    5350           'workdir|w=s'   => \$workdir,
     51           'reduction=s'   => \$reduction,
    5452           'no-update'     => \$no_update,
    5553           'no-op'         => \$no_op,
    … …  
    6765
    6866$ipprc->define_camera($camera);
     67$reduction = 'DEFAULT' unless defined $reduction;
     68my $recipe = $ipprc->reduction($reduction, 'CHIP'); # Recipe to use
    6969
    7070# Look for programs we need
    … …  
    9797
    9898    my $command = "$ppImage -file $input $outputRoot";
    99     $command .= " -recipe PPIMAGE $RECIPE";
     99    $command .= " -recipe PPIMAGE $recipe";
    100100    $command .= " -recipe PPSTATS CHIPSTATS";
    101101    $command .= " -stats $outputStats"; # Command to run ppImage
    … …  
    130130my $bg_stdev = ($stats->bg_stdev() or 'NAN');
    131131my $bg_mean_stdev = ($stats->bg_mean_stdev() or 'NAN');
    132 my $fringe_0      = (${$stats->fringe()[0]} or 'NAN');
    133 my $fringe_1      = (${$stats->fringe_err()[0]} or 'NAN');
     132my $fringe_0      = (${$stats->fringe()}[0] or 'NAN');
     133my $fringe_1      = (${$stats->fringe_err()}[0] or 'NAN');
    134134my $fringe_2      = 'NAN';
    135135

  • trunk/ippScripts/scripts/detrend_process_imfile.pl

    r13748 r13937  
    3232use Pod::Usage qw( pod2usage );
    3333
    34 my ($det_id, $exp_tag, $class_id, $det_type, $input_uri, $camera, $dbname, $workdir, $no_update, $no_op);
     34my ($det_id, $exp_tag, $class_id, $det_type, $input_uri, $camera, $dbname, $workdir, $reduction, $no_update, $no_op);
    3535GetOptions(
    3636    'det_id|d=s'        => \$det_id,
    … …  
    4242    'dbname|d=s'        => \$dbname, # Database name
    4343    'workdir|w=s'       => \$workdir, # Working directory, for output files
     44    'reduction=s'       => \$reduction, # Reduction class
    4445    'no-update'         => \$no_update,
    4546    'no-op'             => \$no_op,
    … …  
    6061$ipprc->define_camera($camera);
    6162
    62 # Recipes to use, as a function of the detrend type
    63 use constant RECIPES => {
    64     'bias'     => 'PPIMAGE_O',    # Overscan only
    65     'dark'     => 'PPIMAGE_OB',   # Overscan and bias only
    66     'shutter'  => 'PPIMAGE_OBD',  # Overscan, bias and dark only
    67     'flat'     => 'PPIMAGE_OBDS', # Overscan, bias, dark and shutter only
    68     'domeflat' => 'PPIMAGE_OBDS', # Overscan, bias, dark and shutter only
    69     'skyflat'  => 'PPIMAGE_OBDS', # Overscan, bias, dark and shutter only
    70     'fringe'   => 'PPIMAGE_OBDSF',# Overscan, bias, dark, shutter and flat only
    71 };
     63$reduction = "DETREND" unless defined $reduction;
     64my $recipe = $ipprc->reduction($reduction, $det_type . '_PROCESS'); # Recipe name to use
    7265
    7366# Look for programs we need
    … …  
    8073}
    8174$ppImage .= " -dbname $dbname" if defined $dbname;
    82 
    83 # Recipe to use in processing
    84 my $recipe = RECIPES->{lc($det_type)};
    85 unless (defined $recipe) {
    86     warn("Unrecognised detrend type: $det_type");
    87     exit($PS_EXIT_CONFIG_ERROR);
    88 }
    8975
    9076$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir;

  • trunk/ippScripts/scripts/detrend_resid.pl

    r13926 r13937  
    3434
    3535my ($det_id, $iter, $exp_tag, $class_id, $det_type, $detrend,
    36         $input_uri, $camera, $mode, $dbname, $workdir, $no_update, $no_op);
     36        $input_uri, $camera, $mode, $dbname, $workdir, $reduction, $no_update, $no_op);
    3737GetOptions(
    3838    'det_id|d=s'        => \$det_id,
    … …  
    4747    'dbname|d=s'        => \$dbname, # Database name
    4848    'workdir|w=s'       => \$workdir,   # Working directory, for output files
     49    'reduction=s'       => \$reduction, # Reduction class
    4950    'no-update'         => \$no_update,
    5051    'no-op'             => \$no_op,
    … …  
    6768$ipprc->define_camera($camera);
    6869
    69 # Recipes to use, as a function of the detrend type
    70 use constant RECIPES => {
    71     'master' => {               # We're creating a master --- already processed the input
    72         'bias'     => 'PPIMAGE_B',      # Bias only
    73         'dark'     => 'PPIMAGE_D',      # Dark only
    74         'shutter'  => 'PPIMAGE_S',      # Shutter only
    75         'flat'     => 'PPIMAGE_F',      # Flat-field only
    76         'domeflat' => 'PPIMAGE_F',      # Flat-field only
    77         'skyflat'  => 'PPIMAGE_F',      # Flat-field only
    78         'fringe'   => 'PPIMAGE_R',      # Fringe only
    79     },
    80     'verify' => {               # We're checking the master --- input is not already processed
    81         'bias'     => 'PPIMAGE_OB',     # Bias only
    82         'dark'     => 'PPIMAGE_OBD',    # Dark only
    83         'shutter'  => 'PPIMAGE_OBDS',   # Shutter only
    84         'flat'     => 'PPIMAGE_OBDSF',  # Flat-field only
    85         'domeflat' => 'PPIMAGE_OBDSF',  # Flat-field only
    86         'skyflat'  => 'PPIMAGE_OBDSF',  # Flat-field only
    87         'fringe'   => 'PPIMAGE_OBDSFR', # Fringe only
    88     },
    89 };
     70$reduction = 'DETREND' unless defined $reduction;
     71my $recipe;                     # Name of recipe to use
     72if ($mode eq 'master') {
     73    $recipe = $det_type . '_RESID';
     74} elsif ($mode eq 'verify') {
     75    $recipe = $det_type . '_VERIFY';
     76} else {
     77    &my_die("Unrecognised mode: $mode", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
     78}
     79$recipe = $ipprc->reduction($reduction, $recipe);
    9080
    9181# Flags to specify the particular detrend to use
    … …  
    113103$ppImage .= " -dbname $dbname" if defined $dbname;
    114104
    115 # Recipe to use in processing
    116 &my_die("Unrecognised mode: $mode", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR) unless defined RECIPES->{$mode};
    117 my $recipe = RECIPES->{$mode}->{lc($det_type)};
    118 &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR) unless defined $recipe;
    119 
    120105$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir;
    121106
    … …  
    183168my $bg_mean_stdev = ($stats->bg_mean_stdev() or 'NAN');
    184169my $bin_stdev     = ($binnedStats->bg_stdev() or 'NAN');
    185 my $fringe_0      = (${$stats->fringe()[0]} or 'NAN');
    186 my $fringe_1      = (${$stats->fringe_err()[0]} or 'NAN');
     170my $fringe_0      = (${$stats->fringe()}[0] or 'NAN');
     171my $fringe_1      = (${$stats->fringe_err()}[0] or 'NAN');
    187172my $fringe_2      = 'NAN';
    188173

  • trunk/ippScripts/scripts/detrend_stack.pl

    r13748 r13937  
    3333use Pod::Usage qw( pod2usage );
    3434
    35 my ($det_id, $iter, $class_id, $det_type, $camera, $dbname, $workdir, $no_update, $no_op, $quiet);
     35my ($det_id, $iter, $class_id, $det_type, $camera, $dbname, $workdir, $reduction, $no_update, $no_op, $quiet);
    3636GetOptions(
    3737    'det_id|d=s'        => \$det_id,
    … …  
    4242    'dbname|d=s'        => \$dbname, # Database name
    4343    'workdir|w=s'       => \$workdir,   # Working directory, for output files
     44    'reduction=s'       => \$reduction, # Reduction class for processing
    4445    'no-update'         => \$no_update,
    4546    'no-op'             => \$no_op,
    … …  
    5859if (defined $quiet) { $verbose = 0; }
    5960
     61$ipprc->define_camera($camera);
     62
    6063# Recipes to use as a function of detrend type
    61 use constant RECIPES => {
    62     'bias' => 'PPMERGE_BIAS',
    63     'dark' => 'PPMERGE_DARK',
    64     'shutter' => 'PPMERGE_SHUTTER',
    65     'flat' => 'PPMERGE_FLAT',
    66     'domeflat' => 'PPMERGE_FLAT',
    67     'skyflat' => 'PPMERGE_FLAT',
    68     'fringe' => 'PPMERGE_FRINGE',
    69     };
     64$reduction = "DETREND" unless defined $reduction;
     65my $recipe = $ipprc->reduction($reduction, $det_type . '_STACK'); # Recipe name to use
    7066
    7167# Look for programs we need
    … …  
    7773    exit($PS_EXIT_CONFIG_ERROR);
    7874}
    79 
    80 my $recipe = RECIPES()->{lc($det_type)}; # Recipe to use in stacking
    81 &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR) unless defined $recipe;
    8275
    8376my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files

  • trunk/ippScripts/scripts/ipp_serial_chip.pl

    r12800 r13937  
    5454    my $class_id = $item->{class_id};
    5555    my $uri = $item->{uri};
     56    my $reduction = $item->{reduction};
    5657   
    5758    my $command = "$chip --chip_id $chip_id --exp_id $exp_id --class_id $class_id --uri $uri --dbname $dbname --camera $camera";
     59    $command .= " --reduction $reduction" if defined $reduction;
    5860    $command .= " --no-op" if defined $no_op;
    5961    $command .= " --no-update" if defined $no_update;

  • trunk/ippScripts/scripts/ipp_serial_detrend.pl

    r12800 r13937  
    6565                my $camera = $item->{camera};
    6666                my $workdir = $item->{workdir};
    67                
     67                my $reduction = $item->{reduction};
     68
    6869                my $command = "$detrend_process_imfile --det_id $det_id --exp_tag $exp_tag --class_id $class_id --det_type $det_type --input_uri $uri --camera $camera --dbname $dbname";
     70                $command .= " --reduction $reduction" if defined $reduction;
    6971                $command .= " --workdir $workdir" if defined $workdir;
    7072                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    119121        my $camera = $item->{camera};
    120122        my $workdir = $item->{workdir};
     123        my $reduction = $item->{reduction};
    121124
    122125        my $command = "$detrend_stack --det_id $det_id --iteration $iteration --class_id $class_id --det_type $det_type --camera $camera --dbname $dbname";
     126        $command .= " --reduction $reduction" if defined $reduction;
    123127        $command .= " --workdir $workdir" if defined $workdir;
    124128        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    235239                my $mode = $item->{mode};
    236240                my $workdir = $item->{workdir};
     241                my $reduction = $item->{reduction};
    237242
    238243                my $command = "$detrend_resid --det_id $det_id --iteration $iteration --exp_tag $exp_tag --class_id $class_id --det_type $det_type --camera $camera --input_uri $uri --mode $mode --dbname $dbname";
     244                $command .= " --reduction $reduction" if defined $reduction;
    239245                $command .= " --detrend $detrend" if defined $detrend;
    240246                $command .= " --workdir $workdir" if defined $workdir;
Note: See TracChangeset for help on using the changeset viewer.