IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 34393


Ignore:
Timestamp:
Sep 5, 2012, 3:47:31 PM (14 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20120805
Files:
1 deleted
16 edited
12 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20120805/DataStore/lib/DataStore/Utils.pm

    r31937 r34393  
    3838);
    3939
    40 $STD_FIELD = qr/^[A-z0-9-_.: ]+$/;
     40$STD_FIELD = qr/^[A-z0-9-+_.: ]+$/;
    4141$TIME_FIELD = qr/^(\d{4})-(\d\d)-(\d\d) T (\d\d):(\d\d):(\d\d) Z$/x;
    4242$BYTE_FIELD = qr/^\d+$/;

  • branches/eam_branches/ipp-20120805/ippMonitor

  • branches/eam_branches/ipp-20120805/ippMonitor/def/failedChipProcessedImfile.d

    r32028 r34393  
    2020ARGS  ARG3 class=$chipProcessedImfile.class_id
    2121
    22 OP    OP1  ( $chipProcessedImfile.fault == 2 ? "chip.revert.on and chip.revert.off" : "chiptool -revertprocessedimfile -fault $chipProcessedImfile.fault -label $chipRun.label -dbname ". $ID['proj'] )
     22OP    OP1  ( $chipProcessedImfile.fault == 2 ? "(chip.revert.on and chip.revert.off) or chiptool -revertprocessedimfile -fault $chipProcessedImfile.fault -label $chipRun.label -dbname ". $ID['proj'] : "chiptool -revertprocessedimfile -fault $chipProcessedImfile.fault -label $chipRun.label -dbname ". $ID['proj'] )
    2323
    2424#     field                            size  format  name         show    link to                          extras

  • branches/eam_branches/ipp-20120805/ippMonitor/def/mopsStatus.d

    r33846 r34393  
    1 TABLE rawExp LEFT JOIN chipRun using(exp_id) LEFT JOIN camRun using(chip_id) LEFT JOIN fakeRun using(cam_id) LEFT JOIN warpRun using(fake_id) LEFT JOIN diffInputSkyfile AS diffInput ON diffInput.warp1 = warp_id LEFT JOIN diffRun AS diffInputRun ON diffInput.diff_id = diffInputRun.diff_id LEFT JOIN diffInputSkyfile AS diffTemplate ON diffTemplate.warp2 = warp_id LEFT JOIN diffRun AS diffTemplateRun ON diffTemplate.diff_id = diffTemplateRun.diff_id LEFT JOIN publishRun ON stage_id=diffInput.diff_id AND (client_id=1 OR client_id=5) LEFT JOIN publishRun AS pubRun2 ON pubRun2.stage_id=diffTemplate.diff_id AND (pubRun2.client_id=1 OR pubRun2.client_id=5) LEFT JOIN magicDSRun ON magicDSRun.magic_ds_id=camRun.magicked
     1TABLE rawExp LEFT JOIN chipRun using(exp_id) LEFT JOIN camRun using(chip_id) LEFT JOIN fakeRun using(cam_id) LEFT JOIN warpRun using(fake_id) LEFT JOIN diffInputSkyfile AS diffInput ON diffInput.warp1 = warp_id LEFT JOIN diffRun AS diffInputRun ON diffInput.diff_id = diffInputRun.diff_id LEFT JOIN diffInputSkyfile AS diffTemplate ON diffTemplate.warp2 = warp_id LEFT JOIN diffRun AS diffTemplateRun ON diffTemplate.diff_id = diffTemplateRun.diff_id LEFT JOIN publishRun ON stage_id=diffInput.diff_id AND (client_id=1 OR client_id=5) LEFT JOIN publishRun AS pubRun2 ON pubRun2.stage_id=diffTemplate.diff_id AND (pubRun2.client_id=1 OR pubRun2.client_id=5)
    22TITLE MOPS Status
    33FILE mopsStatus.php
    … …  
    66
    77UNRESTRICTED WHERE dateobs>=DATE(NOW()) AND rawExp.obs_mode!='ENGINEERING' AND rawExp.obs_mode!='STS' AND rawExp.obs_mode!='M31' AND rawExp.obs_mode!='CNP'
     8RESTRICT_COUNT WHERE dateobs>=DATE(NOW()) AND rawExp.obs_mode!='ENGINEERING' AND rawExp.obs_mode!='STS' AND rawExp.obs_mode!='M31' AND rawExp.obs_mode!='CNP'
    89
    910# OP  OP1   ($ddiff_id > 0 ? $ddiff_id : $tdiff_id)
    … …  
    1213# $diffInput.diff_id+$diffTemplate.diff_id
    1314# PUB ID
    14 OP OP2 ($row[12]>$row[13]?$row[12]. "(I)":($row[13]!=0?$row[13]. "(T)":"0"))
    15 #($row[13]>$row[14])
     15OP OP2 ($row[10]>$row[11]?$row[10]. "(I)":($row[11]!=0?$row[11]. "(T)":"0"))
     16#($row[11]>$row[12])
    1617
    1718#     field                    width format  name          show         link to                   extras
    … …  
    2021FIELD rawExp.exp_id,           5,    %d,     Exp ID
    2122FIELD chipRun.data_group,      5,    %s,     Data grp
    22 FIELD chip_id,                 5,    %d,     Chip ID (Stage 1)
    23 FIELD camRun.cam_id AS camID,                  5,    %d,     Cam ID (Stage 2)
    24 FIELD fake_id,                 5,    %d,     Fake ID (Stage 3)
    25 FIELD warp_id,                 5,    %d,     Warp ID (Stage 4)
     23FIELD chip_id,                 5,    %d,     Chip ID (1/6)
     24FIELD camRun.cam_id AS camID,                  5,    %d,     Cam ID (2/6)
     25FIELD fake_id,                 5,    %d,     Fake ID (3/6)
     26FIELD warp_id,                 5,    %d,     Warp ID (4/6)
    2627FIELD diffInput.diff_id AS ddiff_id,       5,    %d,     ddiff_id, none
    2728FIELD diffTemplate.diff_id AS tdiff_id,    5,    %d,     tdiff_id, none
    28 FIELD *,                       5,    %s,     Diff ID (Stage 5),         op=OP1
    29 FIELD camRun.magicked,         5,    %d,     Magick ID (Stage 6)
    30 FIELD magicDSRun.magic_ds_id,       5,    %d, DestreakID (Stage 7)
     29FIELD *,                       5,    %s,     Diff ID (5/6),             op=OP1
    3130FIELD publishRun.pub_id AS pubId1,       5,    %d,     pubId1, none
    3231FIELD pubRun2.pub_id AS pubId2,       5,    %d,     pubId2, none
    33 FIELD *,                        10, %s, Pub ID, op=OP2
     32FIELD *,                        10, %s, Pub ID (6/6), op=OP2
    3433FIELD filter,                  5,    %s,     Filter
    3534FIELD object,                  15,    %s,     Object

  • branches/eam_branches/ipp-20120805/ippMonitor/raw/czartool_labels.php

    r34249 r34393  
    169169echo "<tr><td>Documentation</td><td><a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\"><font color=\"blue\">here</font></a></td></tr>";
    170170echo "<tr><td>Cluster load</td><td><a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&hc=6&c=IPP%2520Production\"><font color=\"blue\">here</font></a> <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?m=cpu_report&r=hour&s=by%2520name&c=IPP+Production&h=&sh=1&hc=8&z=small\"><font color=\"blue\">cpu_report</font></a> <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?m=mem_report&r=hour&s=by%2520name&c=IPP+Production&h=&sh=1&hc=8&z=small\"><font color=\"blue\">mem_report</font></a></td></tr>";
    171 echo "<tr><td>Who's using the cluster?</td><td><a href=\"http://ippmonitor.ipp.ifa.hawaii.edu/ippMonitor/clusterMonitor/top.html\"><font color=\"blue\">here</font></a></td></tr>";
     171echo "<tr><td>Who's using the cluster?</td><td><a href=\"http://ippmonitor.ipp.ifa.hawaii.edu/ippMonitor/clusterMonitor/top.html\"><font color=\"blue\">here</font></a> <a href=\"http://ippmonitor.ipp.ifa.hawaii.edu/ippMonitor/clusterMonitor2\"><font color=\"blue\">new</font></td></tr>";
    172172echo "<tr><td>Files monitor</td><td><a href=\"http://ippmonitor.ipp.ifa.hawaii.edu/diskMonitor\"><font color=\"blue\">here</font></a></td></tr>";
    173173echo "<tr><td>IPP metrics</td><td><a href=\"http://ipp0022.ifa.hawaii.edu/ps1sc/ippMetrics\"><font color=\"blue\">here</font></a></td></tr>";

  • branches/eam_branches/ipp-20120805/ippMonitor/supervision/check_if_mysql_slave_is_running_fine.py

    r34017 r34393  
    3535
    3636    # Don't change anything after this
    37     db = MySQLdb.connect('localhost', 'root')
     37    db = MySQLdb.connect('localhost', 'root', connect_timeout=1000)
    3838    cursor = db.cursor()
    3939    cursor.execute('SHOW SLAVE STATUS')

  • branches/eam_branches/ipp-20120805/ippScripts/scripts/destreak_restore_camera.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20120805/ippScripts/scripts/ipp_apply_burntool_single.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20120805/ippScripts/scripts/lap_science.pl

    r34064 r34393  
    280280    $warptool_update_cmd   .= " -dbname $dbname " if defined $dbname;
    281281
    282     ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    283         run(command => $chiptool_update_cmd, verbose => $verbose);
    284     unless ($success) {
    285         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    286         &my_die("Unable to perform chiptool -setimfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
    287     }
    288 #     if ($chip_magicDS_id != 0) {   
    289 #       ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    290 #           run(command => $magicDS_update_cmd, verbose => $verbose);
    291 #       unless ($success) {
    292 #           $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    293 #           &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
    294 #       }
    295 #     }
    296 
    297     ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    298         run(command => $warptool_update_cmd, verbose => $verbose);
    299     unless ($success) {
    300         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    301         &my_die("Unable to perform warptool -setskyfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
    302     }
    303    
     282    # only need to update the data if the warps are not full
     283    if ($warp->{state} ne 'full') {
     284        if ($chip->{state} eq 'cleaned') {
     285            ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     286                run(command => $chiptool_update_cmd, verbose => $verbose);
     287            unless ($success) {
     288                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     289                &my_die("Unable to perform chiptool -setimfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
     290            }
     291
     292#             if ($chip_magicDS_id != 0) {   
     293#               ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     294#                   run(command => $magicDS_update_cmd, verbose => $verbose);
     295#               unless ($success) {
     296#                   $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     297#                   &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
     298#               }
     299#             }
     300        }
     301
     302        ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     303            run(command => $warptool_update_cmd, verbose => $verbose);
     304        unless ($success) {
     305            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     306            &my_die("Unable to perform warptool -setskyfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
     307        }
     308    }
    304309    $exposure->{active} = 1;
     310
    305311    return($exposure);
    306312}
    … …  
    309315    my $exposure = shift;
    310316   
     317    if (S64_IS_NOT_NULL($exposure->{warp_id})) {
     318        if ($exposure->{warpRun_state} eq 'full') {
     319            # this warp is in full state no need to update it or the chips
     320            return 0;
     321        }
     322    }
    311323    if (S64_IS_NOT_NULL($exposure->{chip_id})) {
    312324        if (($exposure->{chip_state} eq 'cleaned')||
    … …  
    762774            }
    763775            print "STATUS: Made $Nstacks_made final stacks.\n";
    764             exit($Nstacks_made);
     776            exit(0);
    765777            # If this doesn't do anything, we may end up stuck here.  Therefore, we need a more robust test.
    766778        }

  • branches/eam_branches/ipp-20120805/ippScripts/scripts/magic_destreak.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20120805/ppBackground/src/ppBackgroundRestore.c

    r28300 r34393  
    5959        psImage *patternImage = patternRO->image; // Image with pattern
    6060        psImage *patternMask = patternRO->mask;   // Mask for pattern
     61        // The sign is flipped for the continuity correction.
     62        bool mdok;
     63        bool isContinuity = psMetadataLookupBool(&mdok, pattern->hdu->header, "PTRN_CON");
     64       
    6165        for (int y = 0; y < numRows; y++) {
    6266            for (int x = 0; x < numCols; x++) {
    63                 image->data.F32[y][x] -= patternImage->data.F32[y][x];
     67                if (isContinuity) {
     68                    image->data.F32[y][x] += patternImage->data.F32[y][x];
     69                } else {
     70                    image->data.F32[y][x] -= patternImage->data.F32[y][x];
     71                }
    6472                mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= patternMask->data.PS_TYPE_IMAGE_MASK_DATA[y][x];
    6573            }

  • branches/eam_branches/ipp-20120805/psastro/src/psastroChooseRefstars.c

    r26259 r34393  
    6262
    6363    int nMax = psMetadataLookupS32 (&status, recipe, "PSASTRO.MAX.NREF");
     64    psF32 clampMagMin = psMetadataLookupS32 (&status, recipe, "REFSTAR_CLAMP_MAG_MIN");
     65    if (!status) clampMagMin = -5;
    6466
    6567    // de-activate all files except PSASTRO.REFSTARS
    … …  
    103105                for (int i = 0; i < refs->n; i++) {
    104106                    pmAstromObj *ref = pmAstromObjCopy(refs->data[index->data.S32[i]]);
     107
     108                    if (ref->Mag < clampMagMin) {
     109                        psWarning("Skipping refstar with abusrd magnintude %f ra: %f dec: %f\n",
     110                            ref->Mag, RAD_TO_DEG(ref->sky->r), RAD_TO_DEG(ref->sky->d));
     111                        goto skip;
     112                    }
    105113
    106114                    psProject (ref->TP, ref->sky, fpa->toSky);

  • branches/eam_branches/ipp-20120805/pstamp/scripts

  • branches/eam_branches/ipp-20120805/pstamp/scripts/dqueryparse.pl

    r33380 r34393  
    132132        print STDERR "skipping $command\n";
    133133    }
    134     exit (0);
    135134}
    136135

  • branches/eam_branches/ipp-20120805/tools/dlapstacks

    r33926 r34393  
    1313# allow mask and variance images to be displayed as well
    1414
    15 my $label   = 'LAP.ThreePi.20110809';
     15my $label;
     16my $label1  = 'LAP.ThreePi.20120706';
     17my $label2  = 'LAP.ThreePi.20110809';
    1618my $tess_id = 'RINGS.V3';
    1719
    … …  
    5557$ra = shift;
    5658$dec = shift;
     59# if neither label or data_group were supplied set default label
     60if (!$label and !$data_group) {
     61    $label = $label1;
     62} else {
     63    # drop label2
     64    $label2 = undef;
     65}
     66   
    5767
    5868pod2usage( -msg => 'if ra is suplied <dec> must be supplied as well', -exitval => 2)
    … …  
    90100# don't allow MD label unless it's a refstack.
    91101# Nightly stacks have too many images
    92 if (($data_group && ($data_group =~ /MD/)) or ($label =~ /^MD/)) {
     102if (($data_group && ($data_group =~ /MD/)) or (defined $label and $label =~ /^MD/)) {
    93103    unless ($data_group or uc($label) =~ /REF/) {
    94104        die "cannot use this program for label $label without data_group because that will most likely select nightly stacks and we will match an unreasonably large number of matches. Talk to Bill\n";
    … …  
    124134if ($data_group) {
    125135    $query .= " AND data_group = '$data_group'";
     136} elsif ($label2) {
     137    $query .= " AND (label = '$label' OR label = '$label2')";
    126138} else {
    127139    $query .= " AND label = '$label'";

  • branches/eam_branches/ipp-20120805/tools/listcmf

    r33967 r34393  
    9494}
    9595
     96my $flags_col;
     97my $flags2_col;
    9698
    9799my %extensions;
    … …  
    225227    extend_incomplete_lines(\@lines, $total_cols, $ncols_previous);
    226228} else {
    227     print STDERR "no xfit extension found in $cmf\n" unless $xfit_ext;
     229    print STDERR "no xfit extension found in $cmf\n" if $verbose;
    228230}
    229231
    230232my $base = basename($cmf) ;
     233# data to be output for "missing" sources
     234my $null_line = "";
    231235{
    232236    print $oh "# cmf: $base       FILTER: $filter       ZPT: $zero_point       EXPTIME: $exp_time\n";
    … …  
    235239
    236240    print $oh "IPP_IDET ";
     241    # Loop over columns and print out the header line and build $null_line
    237242    for (my $i = 1; $i < scalar @psf_cols; $i++) {
    238243        print $oh " $psf_cols[$i]_$filter";
     244        if ($i eq $flags_col) {
     245            # for no source set flags to zero
     246            $null_line .= '     0';
     247        } elsif ($i eq $flags2_col) {
     248            # for no source use PM_SOURCE_MODE2_DIFF_WITH_DOUBLE as a "no source" flag
     249            # isnan(X_PSF_$filter) also works but this should be faster to process
     250            $null_line .= '     2';
     251        } elsif ($i > 1) {
     252            # for other columns just print NULL which mana converts to nan
     253            # Note data for column 1, IPP_IDET == $linenum, is added for the null line below
     254            $null_line .= ' NULL';
     255        }
    239256    }
    240257    for (my $i = 0; $i < scalar @xsrc_cols; $i++) {
    241         print $oh " $xsrc_cols[$i]_$filter"
     258        print $oh " $xsrc_cols[$i]_$filter";
     259        $null_line .= ' NULL';
    242260    }
    243261    for (my $i = 0; $i < scalar @xfit_cols; $i++) {
    244262        print $oh " $xfit_cols[$i]_$filter";
     263        $null_line .= ' NULL';
    245264    }
    246265    print $oh "\n";
    247266}
    248267
    249 foreach my $line (@lines) {
     268my $num_lines = scalar @lines;
     269for (my $i = 1; $i < $num_lines; $i++) {
     270    my $line = $lines[$i];
     271    if (!$line) {
     272        # Add line for missing source
     273        $line = sprintf "%11d %s", $i, $null_line;
     274    }
    250275    print $oh "$line\n";
    251276}
    … …  
    323348            # XXX TODO: check the format and insure that it is not a vector
    324349            # because we currently don't support them
     350            if ($name eq 'FLAGS') {
     351                # print STDERR "found flags in column $col\n";
     352                $flags_col = $col;
     353            }
     354            if ($name eq 'FLAGS2') {
     355                # print STDERR "found flags2 in column $col\n";
     356                $flags2_col = $col;
     357            }
    325358        }
    326359        push @$r_cols, $name;
    … …  
    336369        # XXX: we are assuming here that the ids in a psf extension
    337370        # are sequential
    338         foreach my $line (@lines) {
     371        # foreach my $line (@lines) {
     372        my $num_lines = scalar @lines;
     373        for (my $i = 0; $i < $num_lines; $i++) {
     374            my $line = $lines[$i];
    339375            # id is first column
    340376            chomp $line;
    341377            my ($ipp_idet) = split " ", $line;
    342             push @$r_ids, $ipp_idet;
    343             push @$r_lines, $line;
     378#            push @$r_ids, $ipp_idet;
     379#            push @$r_lines, $line;
     380            $r_ids->[$ipp_idet] = $ipp_idet;
     381            $r_lines->[$ipp_idet] = $line;
    344382        }
    345383    } else {
    346         my $num_objects = scalar @$r_lines;
     384        my $num_lines = scalar @$r_lines;
    347385        foreach my $line (@lines) {
    348386            chomp $line;
    349387            my ($id, $rest_of_line) = identify_line($line);
    350             if ($id > $num_objects) {
    351                 die "TILT: id $id is greater than num objects\n";
     388            if ($id > $num_lines) {
     389#                die "TILT: id $id is greater than num objects\n";
    352390            }
    353391            # just append the line to this object's line
    … …  
    374412    my $nprevious = shift;
    375413    for (my $i = 0; $i < scalar @$lines; $i++) {
     414        next if !$lines->[$i];
    376415        my @words = split " ", $lines->[$i];
    377416        my $nwords = scalar @words;
Note: See TracChangeset for help on using the changeset viewer.