IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 35350


Ignore:
Timestamp:
Apr 2, 2013, 3:01:21 PM (13 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20130307
Files:
81 edited
9 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20130307

  • branches/eam_branches/ipp-20130307/Nebulous

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/Nebulous-Server

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/Ohana

  • branches/eam_branches/ipp-20130307/Ohana/src/delstar/src/delstar.c

    r35237 r35350  
    2121  switch (MODE) {
    2222    case MODE_DUP_IMAGES:
    23       delete_duplicate_images (0, NULL, &db);
     23      if (!delete_duplicate_images (0, NULL, &db)) exit (1);
     24      exit (0);
    2425      break;
    2526    case MODE_IMAGEFILE:

  • branches/eam_branches/ipp-20130307/Ohana/src/libohana/src

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/Ohana/src/opihi

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/Ohana/src/opihi/cmd.astro

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/Ohana/src/opihi/cmd.data

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/Ohana/src/relastro/src

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

    r34716 r35350  
    223223    my $choose_components = 0;
    224224    if ($stage eq "raw") {
    225         $command = "$regtool -processedimfile -dbname $imagedb";
     225        if ($component or $use_imfile_id or !$skycenter) {
     226            $command = "$regtool -processedimfile -dbname $imagedb";
     227            if ($component and $component ne 'all') {
     228                $class_id = $component;
     229                $component_args = " -class_id $class_id";
     230            }
     231        } else {
     232            $command = "$regtool -processedexp -dbname $imagedb";
     233            $choose_components = 1;
     234        }
    226235        # XXX: for now restrict lookups to type object
    227236        # are stamps of detrend exposures interesting?

  • branches/eam_branches/ipp-20130307/dbconfig/cam.md

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/dbconfig/changes.txt

    r35239 r35350  
    23882388CREATE TABLE lapGroup  (
    23892389    seq_id  BIGINT,
     2390    tess_id VARCHAR(64),
    23902391    projection_cell VARCHAR(64),
    23912392    state   VARCHAR(16),
     2393    label   VARCHAR(654),
     2394    registered TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
    23922395    fault   SMALLINT,
    23932396    PRIMARY KEY(seq_id, projection_cell),
    … …  
    23992402UPDATE dbversion set schema_version = '1.1.75', updated= CURRENT_TIMESTAMP();
    24002403
     2404
     2405-- new Postage Stamp Request columns
     2406-- Note: These do not aply to the gpc1 database because the pstamp tables were
     2407-- deleted from that database.
     2408ALTER TABLE pstampRequest ADD COLUMN username VARCHAR(255) AFTER outdir;
     2409ALTER TABLE pstampRequest ADD COLUMN proj_id BIGINT AFTER username;
     2410ALTER TABLE pstampRequest ADD COLUMN registered DATETIME AFTER proj_id;
     2411
     2412-- default to the gpc1 project
     2413UPDATE pstampRequest SET proj_id = 1;

  • branches/eam_branches/ipp-20130307/dbconfig/lap.md

    r35213 r35350  
    3333lapGroup METADATA
    3434    seq_id          S64 0
     35    tess_id         STR 64
    3536    projection_cell STR 64
    3637    state           STR 64
     38    label           STR 64
     39    registered      TAI NULL
    3740    fault           S16 0
    3841end

  • branches/eam_branches/ipp-20130307/dbconfig/pstamp.md

    r33119 r35350  
    3232    uri         STR         255
    3333    outdir      STR         255
     34    username    STR         255
     35    proj_id     S64         0
     36    registered  UTC         0001-01-01T00:00:00Z
    3437    timestamp   UTC         0001-01-01T00:00:00Z
    3538    fault       S32         0

  • branches/eam_branches/ipp-20130307/ippMonitor

  • branches/eam_branches/ipp-20130307/ippMonitor/def/distRun.d

    r32636 r35350  
    88WHERE distRun.dist_id = distComponent.dist_id
    99UNRESTRICTED AND 0=1
     10RESTRICT_COUNT AND 0>1
    1011
    1112#     field                   size  format  name         show    link to                  extras

  • branches/eam_branches/ipp-20130307/ippMonitor/raw/heathers_plot.php

    r35200 r35350  
    1010menu('ipp.czar.dat', 'useful tools for the czar', 'ipp.css', $ID['link'], $ID['proj']);
    1111
     12echo "These images are generated every day at 7:00 HST (17:00 GMT)\n";
    1213// document body
    1314echo "<table>\n";

  • branches/eam_branches/ipp-20130307/ippMonitor/raw/ipp.imfiles.dat

    r35199 r35350  
    2424menutop   | menutop      | link    | czartool - ipptopsps         | czartool_ipptopsps.php
    2525menutop   | menutop      | link    | mask stats                   | maskStats.php
    26 menutop   | menutop      | link    | Heather's Plots              | heathers_plot.php
    2726
    2827menutop   | menutop      | plain   | &nbsp;                       |

  • branches/eam_branches/ipp-20130307/ippMonitor/raw/ipp.plots.dat

    r34434 r35350  
    88menulink  | menuselect      | link    | Simple plot - staticsky             | simplePlotStaticskyImage.php
    99menulink  | menuselect      | link    | Simple plot - skycal             | simplePlotSkycalImage.php
     10menulink  | menuselect      | link    | Heather's Plots              | heathers_plot.php
    1011
    1112menutop   | menutop      | plain   | &nbsp;                                 |

  • branches/eam_branches/ipp-20130307/ippScripts/Build.PL

    r35247 r35350  
    130130        scripts/regenerate_background.pl
    131131        scripts/relgroup_exp_list.pl
     132        scripts/queuestaticsky.pl
    132133    )],
    133134    dist_abstract => 'Scripts for running the Pan-STARRS IPP',

  • branches/eam_branches/ipp-20130307/ippScripts/scripts/chip_imfile.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippScripts/scripts/destreak_restore_camera.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippScripts/scripts/detrend_process_imfile.pl

    r34800 r35350  
    129129    my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
    130130    unless ($repair_success) {
    131         &my_die("Unable to attempt repair: $input_uri $repair_error_code", $det_id, $exp_id, $class_id $PS_EXIT_SYS_ERROR);
     131        &my_die("Unable to attempt repair: $input_uri $repair_error_code", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
    132132    }
    133133

  • branches/eam_branches/ipp-20130307/ippScripts/scripts/diff_skycell.pl

    r35133 r35350  
    113113my ($inputMagic, $templateMagic); # Are the inputs been magicked?
    114114my ($saveInConv, $saveRefConv);   # Save the input or reference convolved images?
    115 # Prescan to decide if this is or is not a stack stack diff. The check above confirms we only have two entries.
    116 if ((${ $files }[0]->{warp_id} == 0)&&
    117     (${ $files }[1]->{warp_id} == 0)) {
    118     # Both are zero, so stack stack diff;
    119     unless ($use_convolved) {
    120         $use_convolved = 0;
    121     }
    122 }
    123 else {
    124     # We're in some sort of warp stack or warp warp (don't care about the last one.)
    125     $use_convolved = 1;
    126 }
     115# CZW 2013-03-21: We only want to use unconvolved inputs.  Ever, if I understand correctly.
     116unless ($use_convolved) {
     117    $use_convolved = 0;
     118}
     119
     120# # Prescan to decide if this is or is not a stack stack diff. The check above confirms we only have two entries.
     121# if ((${ $files }[0]->{warp_id} != 0)&&
     122#     (${ $files }[1]->{warp_id} != 0)) {
     123#     # Both are zero, so stack stack diff;
     124# }
     125# else {
     126
     127#     # We're in some sort of warp stack or warp warp (don't care about the last one.)
     128#     $use_convolved = 1; ## This is a hack to do a test for mops, and should not be committed.
     129# }
    127130
    128131# ppSub does (input) - (template) after PSF-match convolution. 

  • branches/eam_branches/ipp-20130307/ippScripts/scripts/ipp_apply_burntool_single.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippScripts/scripts/magic_destreak.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippScripts/scripts/mergedvodb_merge.pl

    r33030 r35350  
    156156            $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path";
    157157        } else {
    158             $merge_command = "$dvomerge $minidvodb_path into $mergedvodb_path";
     158            $merge_command = "$dvomerge -parallel $minidvodb_path into $mergedvodb_path";
    159159        }
    160160        print "\n$merge_command\n";

  • branches/eam_branches/ipp-20130307/ippScripts/scripts/minidvodb_premerge.pl

    r33030 r35350  
    1616my $dtime_relphot;
    1717my $dtime_script;
     18my $dtime_delstar;
    1819
    1920use vars qw( $VERSION );
    … …  
    3334my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1);
    3435my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1);
     36my $delstar = can_run('delstar') or (warn "Can't find delstar" and $missing_tools = 1);
    3537my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1);
    3638my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1);
    3739my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1);
     40
    3841my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1);
    3942
    … …  
    8083unless ($no_op) {
    8184   
    82         #this is chopped into several parts: addstar, relphot
     85        #this is chopped into several parts: delstar,addstar, relphot, relastro, dvoverify
     86        #delstar - first step: are there duplicates, if so remove them
     87        {
     88            my $command  = "$delstar -update -dup-images ";
     89            #$command .= " -D CAMERA $camera";
     90            $command .= " -D CATDIR $minidvodb";
     91            my $mjd_delstar_start = DateTime->now->mjd;   # MJD of starting script
     92            print "\n$command\n";
     93            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     94                run(command => $command, verbose => $verbose);
     95            unless ($success) {
     96                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     97                &my_die("Unable to perform delstar: $error_code", $minidvodb_id, $error_code);
     98            }
     99            print $full_buf;
     100            $dtime_delstar = 86400.0*(DateTime->now->mjd - $mjd_delstar_start); 
     101            # MJD of starting script
     102            print "delstar time $dtime_delstar\n";
     103        }
     104
     105
     106
    83107        #addstar
    84108        {
    … …  
    94118                &my_die("Unable to perform addstar: $error_code", $minidvodb_id, $error_code);
    95119            }
    96             $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);  $dtime_resort = $dtime_addstar;
    97             # MJD of starting script
     120            $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); 
     121            # MJD of starting script
    98122            $dtime_resort = $dtime_addstar;
    99123            print "addstar -resort time $dtime_addstar\n";

  • branches/eam_branches/ipp-20130307/ippScripts/scripts/publish_file.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippScripts/scripts/warp_skycell.pl

    r35239 r35350  
    247247    if (($doBackground)) {
    248248        # &&($imfile->{background_model} == 1)
    249         if ($imfile->{cam_background_model}) {
     249        if (($imfile->{cam_background_model})&&($imfile->{cam_background_model} != 32767)) {
    250250            $bkg    = $ipprc->filename("PPIMAGE.BACKMDL", $imfile->{cam_path_base}, $imfile->{class_id});
    251251        }

  • branches/eam_branches/ipp-20130307/ippTasks

  • branches/eam_branches/ipp-20130307/ippTasks/addstar.pro

    r34804 r35350  
    66check.globals
    77
     8
     9if ($?ADDSTAGES:n == 0)       set ADDSTAGES:n = 0
     10
     11
     12
    813book init addPendingExp
    914
    … …  
    1722
    1823macro addstar.on
    19   task addstar.exp.load.stack
    20     active true
    21   end
    22   task addstar.exp.load.cam
    23     active true
    24   end
    25   task addstar.exp.load.staticsky
    26     active true 
    27   end
    28   task addstar.exp.load.skycal
     24  task addstar.exp.load
    2925    active true
    3026  end
    … …  
    3531
    3632macro addstar.off
    37   task addstar.exp.load.stack
    38     active false
    39   end
    40   task addstar.exp.load.cam
    41     active false
    42   end
    43   task addstar.exp.load.staticsky
    44     active false
    45   end
    46   task addstar.exp.load.skycal
     33  task addstar.exp.load
    4734    active false
    4835  end
    … …  
    8370
    8471
     72#addstar stages
     73
     74macro add.addstages
     75  if ($0 != 2)
     76    echo "USAGE: add.addstages (addstages)"
     77    break
     78  end
     79  if ($?ADDSTAGES:n == 0)
     80    list ADDSTAGES -add $1
     81    return
     82  end
     83
     84  local found
     85  $found = 0
     86  for i 0 $ADDSTAGES:n
     87    if ($ADDSTAGES:$i == $1)
     88      $found = 1
     89      echo "$ADDSTAGES:$i set"
     90      last
     91    end
     92  end
     93 
     94  if ($found == 0)
     95    list ADDSTAGES -add $1
     96  end
     97end
     98
     99macro del.addstages
     100  if ($0 != 2)
     101    echo "USAGE: del.addstages (addstages)"
     102    break
     103  end
     104  if ($?ADDSTAGES:n == 0)
     105    return
     106  end
     107
     108  list ADDSTAGES -del $1
     109end
     110
     111macro show.addstages
     112  if ($0 != 1)
     113    echo "USAGE: show.addstages"
     114    break
     115  end
     116  if ($?ADDSTAGES:n == 0)
     117    echo "no addstar stages defined"
     118  end
     119  if ($ADDSTAGES:n == 0)
     120    echo "no addstar stages defined"
     121  end
     122
     123  local i
     124  for i 0 $ADDSTAGES:n
     125    echo $ADDSTAGES:$i
     126  end
     127end
     128
     129macro stuff
     130   echo $LOADEXEC_ADD
     131   echo $LOADPOLL
     132end
     133
    85134# this variable will cycle through the known database names
    86135$addstar_DB = 0
     136$addstar_stages_DB = 0
    87137#this may not work for more databases (addstar) will do that later)
    88138$addstar_revert_DB_C = 0
    … …  
    92142$addstar_revert_DB_SC = 0
    93143
    94 $LOADEXEX_ADD = 300
    95 
     144#$LOADEXEC_ADD = 300
     145$LOADEXEC_ADD = 10
    96146
    97147
    … …  
    99149# new entries are added to addPendingExp
    100150# skip already-present entries
    101 task           addstar.exp.load.stack
     151task           addstar.exp.load
    102152  host         local
    103153
    … …  
    111161
    112162  task.exec
    113    # if ($LABEL:n == 0) break
    114     $run = addtool -pendingexp -stage stack
    115     if ($DB:n == 0)
    116       option DEFAULT
    117     else
    118       # save the DB name for the exit tasks
    119       option $DB:$addstar_DB
    120       $run = $run -dbname $DB:$addstar_DB
    121       $addstar_DB ++
    122       if ($addstar_DB >= $DB:n) set addstar_DB = 0
    123     end
     163    #if no stages defined we can't run
     164    if ($ADDSTAGES:n == 0) echo "what"
     165 #   if ($ADDSTAGES:n == 0) break
     166    $run = addtool -pendingexp
     167   
     168    #option $ADDSTAGES:$addstar_stages_DB
     169    $run = $run -stage $ADDSTAGES:$addstar_stages_DB 
     170    option $DB:$addstar_DB
     171    $run = $run -dbname $DB:$addstar_DB
     172    $addstar_stages_DB ++
     173      if ($addstar_stages_DB >= $ADDSTAGES:n)
     174         set addstar_stages_DB = 0
     175         $addstar_DB ++
     176         if ($addstar_DB >= $DB:n) set addstar_DB = 0
     177         #go to next stage. if run out of stages, go to 0 stage and
     178         #cycle through dbname, if run out of dbnames, go to 0 dbname
     179      end
    124180    add_poll_args run
    125181    add_poll_labels run
     182    echo $run
    126183    command $run
    127184  end
    … …  
    153210  end
    154211end
    155 
    156 task           addstar.exp.load.cam
    157   host         local
    158 
    159   periods      -poll $LOADPOLL
    160   periods      -exec $LOADEXEC_ADD
    161   periods      -timeout 300
    162   npending     1
    163 
    164   stdout NULL
    165   stderr $LOGDIR/addstar.exp.log
    166 
    167   task.exec
    168    # if ($LABEL:n == 0) break
    169     $run = addtool -pendingexp -stage cam
    170     if ($DB:n == 0)
    171       option DEFAULT
    172     else
    173       # save the DB name for the exit tasks
    174       option $DB:$addstar_DB
    175       $run = $run -dbname $DB:$addstar_DB
    176       $addstar_DB ++
    177       if ($addstar_DB >= $DB:n) set addstar_DB = 0
    178     end
    179     add_poll_args run
    180     add_poll_labels run
    181     command $run
    182   end
    183 
    184   # success
    185   task.exit    0
    186     # convert 'stdout' to book format
    187     ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT
    188     if ($VERBOSE > 2)
    189       book listbook addPendingExp
    190     end
    191 
    192     # delete existing entries in the appropriate pantaskStates
    193     process_cleanup addPendingExp
    194   end
    195 
    196   # default exit status
    197   task.exit    default
    198     showcommand failure
    199   end
    200 
    201   task.exit    crash
    202     showcommand crash
    203   end
    204 
    205   # operation times out?
    206   task.exit    timeout
    207     showcommand timeout
    208   end
    209 end
    210 
    211 task           addstar.exp.load.staticsky
    212   host         local
    213   periods      -poll $LOADPOLL
    214   periods      -exec $LOADEXEC_ADD
    215   periods      -timeout 300
    216   npending     1
    217 
    218   stdout NULL
    219   stderr $LOGDIR/addstar.exp.log
    220 
    221   task.exec
    222    # if ($LABEL:n == 0) break
    223     $run = addtool -pendingexp -stage staticsky
    224     if ($DB:n == 0)
    225       option DEFAULT
    226     else
    227       # save the DB name for the exit tasks
    228       option $DB:$addstar_DB
    229       $run = $run -dbname $DB:$addstar_DB
    230       $addstar_DB ++
    231       if ($addstar_DB >= $DB:n) set addstar_DB = 0
    232     end
    233     add_poll_args run
    234     add_poll_labels run
    235     command $run
    236   end
    237 
    238   # success
    239   task.exit    0
    240     # convert 'stdout' to book format
    241     ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT
    242     if ($VERBOSE > 2)
    243       book listbook addPendingExp
    244     end
    245 
    246     # delete existing entries in the appropriate pantaskStates
    247     process_cleanup addPendingExp
    248   end
    249 
    250   # default exit status
    251   task.exit    default
    252     showcommand failure
    253   end
    254 
    255   task.exit    crash
    256     showcommand crash
    257   end
    258 
    259   # operation times out?
    260   task.exit    timeout
    261     showcommand timeout
    262   end
    263 end
    264 
    265 task           addstar.exp.load.skycal
    266   host         local
    267   periods      -poll $LOADPOLL
    268   periods      -exec $LOADEXEC_ADD
    269   periods      -timeout 300
    270   npending     1
    271 
    272   stdout NULL
    273   stderr $LOGDIR/addstar.exp.log
    274 
    275   task.exec
    276    # if ($LABEL:n == 0) break                                                                                 
    277     $run = addtool -pendingexp -stage skycal
    278     if ($DB:n == 0)
    279       option DEFAULT
    280     else
    281       # save the DB name for the exit tasks                                                                   
    282       option $DB:$addstar_DB
    283       $run = $run -dbname $DB:$addstar_DB
    284       $addstar_DB ++
    285       if ($addstar_DB >= $DB:n) set addstar_DB = 0
    286     end
    287     add_poll_args run
    288     add_poll_labels run
    289     command $run
    290   end
    291 
    292   # success                                                                                                   
    293   task.exit    0
    294     # convert 'stdout' to book format                                                                         
    295     ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT
    296     if ($VERBOSE > 2)
    297       book listbook addPendingExp
    298     end
    299 
    300     # delete existing entries in the appropriate pantaskStates                                               
    301     process_cleanup addPendingExp
    302   end
    303 
    304   # default exit status                                                                                       
    305   task.exit    default
    306     showcommand failure
    307   end
    308 
    309   task.exit    crash
    310     showcommand crash
    311   end
    312 
    313   # operation times out?                                                                                     
    314   task.exit    timeout
    315     showcommand timeout
    316   end
    317 end
    318 
    319212
    320213# run the addstar script on pending exposures

  • branches/eam_branches/ipp-20130307/ippTasks/minidvodb.pro

    r33706 r35350  
    145145  book setword MINIDVODB_PREMERGE $1 MINIDVODB_GROUP $1
    146146  book setword MINIDVODB_PREMERGE $1 DVODB $3
     147  book setword MINIDVODB_PREMERGE $1 CAMERA $5
    147148  book setword MINIDVODB_PREMERGE $1 STATE PENDING
    148149  #active 
    … …  
    443444    book getword minidvodbPreMergelist $pageName minidvodb_group -var MINIDVODB_GROUP
    444445    book getword minidvodbPreMergelist $pageName minidvodb_path -var MINIDVODB_PATH
     446    book getword minidvodbPreMergelist $pageName camera -var CAMERA
    445447    book getword minidvodbPreMergelist $pageName state -var STATE
    446448    stdout $LOGDIR/minidvodb.premerge.run.log
    447449    stderr $LOGDIR/minidvodb.premerge.run.log
    448450
    449     $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID
     451    $run = minidvodb_premerge.pl --camera $CAMERA --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID
    450452   
    451453  if ($DB:n == 0)

  • branches/eam_branches/ipp-20130307/ippTasks/survey.pro

    r34976 r35350  
    1818 book create SURVEY_STATICSKYSINGLE
    1919 book create SURVEY_SKYCAL
     20 book create SURVEY_LAPGROUP
    2021 $haveSurveyBooks = TRUE
    2122end
    … …  
    3334$SURVEY_PUBLISH_DB = 0
    3435$SURVEY_SKYCAL_DB = 0
     36$SURVEY_LAPGROUP_DB = 0
    3537$SURVEY_STATICSKYSINGLE_DB = 0
    3638$SURVEY_EXEC = 120
    … …  
    7880    active true
    7981  end
     82  task survey.lapgroup
     83    active true
     84  end
    8085end
    8186
    … …  
    118123  end
    119124  task survey.skycal
     125    active false
     126  end
     127  task survey.lapgroup
    120128    active false
    121129  end
    … …  
    494502macro survey.show.skycal
    495503  if ($0 != 1)
    496     echo "USAGE: survey.show.skyacl"
     504    echo "USAGE: survey.show.skycal"
    497505    break
    498506  end
    499507  book listbook SURVEY_SKYCAL
     508end
     509
     510macro survey.add.lapgroup
     511  if ($0 != 3)
     512    echo "USAGE: survey.add.lapgroup (label) (seq_id)"
     513    break
     514  end
     515  book newpage SURVEY_LAPGROUP $1
     516  book setword SURVEY_LAPGROUP $1 LABEL $1
     517  book setword SURVEY_LAPGROUP $1 SEQ_ID $2
     518  book setword SURVEY_LAPGROUP $1 STATE PENDING
     519end
     520
     521macro survey.del.lapgroup
     522  if ($0 != 2)
     523    echo "USAGE: survey.del.lapgroup (label)"
     524    break
     525  end
     526  book delpage SURVEY_LAPGROUP $1
     527end
     528
     529macro survey.show.lapgroup
     530  if ($0 != 1)
     531    echo "USAGE: survey.show.lapgroup"
     532    break
     533  end
     534  book listbook SURVEY_LAPGROUP
    500535end
    501536
    … …  
    15611596  end
    15621597end
     1598
     1599task survey.lapgroup
     1600  host local
     1601 
     1602  periods      -poll $SURVEY_POLL
     1603  periods      -exec $SURVEY_EXEC
     1604  periods      -timeout $SURVEY_TIMEOUT
     1605  npending     1
     1606
     1607  stdout $LOGDIR/survey.lapgroup.log
     1608  stderr $LOGDIR/survey.lapgroup.log
     1609
     1610  task.exec
     1611    book npages SURVEY_LAPGROUP -var N
     1612    if ($N == 0)
     1613#      echo "No labels for processing"
     1614      break
     1615    endif
     1616
     1617    book getpage SURVEY_LAPGROUP 0 -var label -key STATE NEW
     1618    if ("$label" == "NULL")
     1619      # All labels have been done --- reset
     1620#      echo "Resetting labels"
     1621      for i 0 $N
     1622        book getpage SURVEY_LAPGROUP $i -var label
     1623        book setword SURVEY_LAPGROUP $label STATE NEW
     1624      end
     1625      book getpage SURVEY_LAPGROUP 0 -var label -key STATE NEW
     1626
     1627      # Select different database
     1628      $SURVEY_LAPGROUP ++
     1629      if ($SURVEY_LAPGROUP >= $DB:n) set SURVEY_LAPGROUP = 0
     1630    end
     1631
     1632    book setword SURVEY_LAPGROUP $label STATE DONE
     1633    book getword SURVEY_LAPGROUP $label SEQ_ID -var SEQ_ID
     1634
     1635    # For now the list of filters
     1636    $run = laptool -definegroup -seq_id $SEQ_ID -set_label $label -filter g.00000 -filter r.00000 -filter i.00000 -filter z.00000 -filter y.00000
     1637
     1638    if ($DB:n == 0)
     1639      option DEFAULT
     1640    else
     1641      $run = $run -dbname $DB:$SURVEY_LAPGROUP_DB
     1642      option $DB:$SURVEY_LAPGROUP_DB
     1643    end
     1644   
     1645    # echo $run
     1646    command $run
     1647  end
     1648
     1649  # success
     1650  task.exit    0
     1651#    echo "Success"
     1652  end
     1653
     1654  # locked list
     1655  task.exit    default
     1656    showcommand failure
     1657  end
     1658
     1659  task.exit    crash
     1660    showcommand crash
     1661  end
     1662
     1663  # operation times out?
     1664  task.exit    timeout
     1665    showcommand timeout
     1666  end
     1667end

  • branches/eam_branches/ipp-20130307/ippToPsps

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/cleanup.py

    r35097 r35350  
    7373        self.logger.infoTitle("Cleaning up batch type %s" % batchType)
    7474
    75         loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType)
    76         deleteFromDxLayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType)
    77         mergedLocalIDs = self.ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType)
     75        mergedLocalIDs = self.ippToPspsDb.getMergedReadyToDeleteFromLocalDisk(batchType)
     76        purgedLocalIDs = self.ippToPspsDb.getPurgedReadyToDeleteFromLocalDisk(batchType)
     77        deleteFromLocalIDs = mergedLocalIDs + purgedLocalIDs
    7878
    79         purgedDatastoreIDs = self.ippToPspsDb.getPurgedButNotDeletedFromDatastore(batchType)
    80         purgedLocalIDs = self.ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType)
     79        loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMReadyToDeleteFromDatastore(batchType)
     80        purgedDatastoreIDs = self.ippToPspsDb.getPurgedReadyToDeleteFromDatastore(batchType)
     81        deleteFromDatastoreIDs = loadedDatastoreIDs + purgedDatastoreIDs
    8182
    82         deleteFromDatastoreIDs = loadedDatastoreIDs + purgedDatastoreIDs
    83         deleteFromLocalIDs = mergedLocalIDs + purgedLocalIDs
     83        deleteFromDxLayerIDs = self.ippToPspsDb.getLoadedToODMReadyToDeleteFromDxLayer(batchType)
    8484
    8585        # report to log what we are going to do

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/ipptopspsdb.py

    r35217 r35350  
    3030
    3131    '''
    32     Returns a list of merged batch IDs that are merged but not yet deleted
    33     '''
    34     def getMergedButNotDeletedBatchIDs(self, batchType, column):
    35 
     32    Returns a list of merged batch IDs that are merged but not yet deleted (but allowed to be deleted)
     33    '''
     34    def getMergedReadyToDeleteFromLocalDisk(self, batchType):
     35   
    3636        sql = "SELECT DISTINCT batch_id \
    3737               FROM batch \
    … …  
    4040               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
    4141               AND merged = 1 \
    42                AND " + column + " = 0"
    43 
     42               AND deleted_local = 0 \
     43               AND block_delete_local = 0 "
     44   
    4445        ids = []
    4546        try:
    … …  
    4950        except:
    5051            self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
    51 
     52   
    5253        self.logger.debug("Found %d merged but un-deleted items" % len(ids))
    53 
     54   
    5455        return ids
    5556
    5657    '''
    57     Returns a list of batch IDs marked as 'purged' but not yet deleted
    58     '''
    59     def getPurgedButNotDeletedBatchIDs(self, batchType, column):
    60 
     58    Returns a list of batch IDs marked as 'purged' but not yet deleted (but allowed to be deleted)
     59    '''
     60    def getPurgedReadyToDeleteFromLocalDisk(self, batchType):
     61   
    6162        sql = "SELECT DISTINCT batch_id \
    6263               FROM batch \
    … …  
    6566               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
    6667               AND purged = 1 \
    67                AND " + column + " = 0"
    68 
     68               AND deleted_local = 0 \
     69               AND block_delete_local = 0 "
     70   
    6971        ids = []
    7072        try:
    … …  
    7476        except:
    7577            self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
    76 
    77 
     78   
     79   
    7880        self.logger.debug("Found %d merged but un-deleted items" % len(ids))
    79 
     81   
    8082        return ids
    8183
    8284    '''
    83     Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted
    84     '''
    85     def getLoadedToODMButNotDeletedBatchIDs(self, batchType, column):
    86 
     85    Returns a list of batch IDs marked as 'purged' but not yet deleted (but allowed to be deleted)
     86    '''
     87    def getPurgedReadyToDeleteFromDatastore(self, batchType):
     88   
    8789        sql = "SELECT DISTINCT batch_id \
    8890               FROM batch \
    … …  
    9092               AND batch_type = '" + batchType + "' \
    9193               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
     94               AND purged = 1 \
     95               AND deleted_datastore = 0 \
     96               AND block_delete_datastore = 0 "
     97   
     98        ids = []
     99        try:
     100            rs = self.executeQuery(sql)
     101            while (rs.next()): ids.append(rs.getInt(1))
     102            rs.close()
     103        except:
     104            self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
     105   
     106   
     107        self.logger.debug("Found %d merged but un-deleted items" % len(ids))
     108   
     109        return ids
     110
     111    '''
     112    Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted from the datastore
     113    '''
     114    def getLoadedToODMReadyToDeleteFromDatastore(self, batchType):
     115
     116        sql = "SELECT DISTINCT batch_id \
     117               FROM batch \
     118               WHERE timestamp > '" + self.skychunk.epoch + "' \
     119               AND batch_type = '" + batchType + "' \
     120               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
    92121               AND (loaded_to_ODM = -1 OR merge_worthy = 1) \
    93                AND " + column + " = 0"
     122               AND deleted_datastore = 0 AND block_delete_datastore = 0"
    94123
    95124        ids = []
    … …  
    106135
    107136    '''
    108     Returns a list of merged batch IDs that are not deleted from local disk
    109     '''
    110     def getMergedButNotDeletedFromLocalDisk(self, batchType):
    111         return self.getMergedButNotDeletedBatchIDs(batchType, "deleted_local")
    112 
    113     '''
    114     Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from local disk
    115     '''
    116     def getLoadedToODMButNotDeletedFromLocalDisk(self, batchType):
    117         return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_local")
    118 
    119     '''
    120     Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from datastore
    121     '''
    122     def getLoadedToODMButNotDeletedFromDatastore(self, batchType):
    123         return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_datastore")
    124 
    125     '''
    126     Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from DXLayer
    127     '''
    128     def getLoadedToODMButNotDeletedFromDXLayer(self, batchType):
    129         return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_dxlayer")
    130 
    131     '''
    132     Returns a list of purged  batch IDs that not deleted from local disk
    133     '''
    134     def getPurgedButNotDeletedFromLocalDisk(self, batchType):
    135         return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_local")
    136 
    137     '''
    138     Returns a list of purged batch IDs that are not deleted from datastore
    139     '''
    140     def getPurgedButNotDeletedFromDatastore(self, batchType):
    141         return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_datastore")
     137    Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted from the dxlayer
     138    '''
     139    def getLoadedToODMReadyToDeleteFromDxLayer(self, batchType):
     140
     141        sql = "SELECT DISTINCT batch_id \
     142               FROM batch \
     143               WHERE timestamp > '" + self.skychunk.epoch + "' \
     144               AND batch_type = '" + batchType + "' \
     145               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
     146               AND (loaded_to_ODM = -1 OR merge_worthy = 1) \
     147               AND deleted_dxlayer = 0 and block_delete_dxlayer = 0"
     148
     149        ids = []
     150        try:
     151            rs = self.executeQuery(sql)
     152            while (rs.next()): ids.append(rs.getInt(1))
     153            rs.close()
     154        except:
     155            self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
     156
     157        self.logger.debug("Found %d merged but un-deleted items" % len(ids))
     158
     159        return ids
     160
     161    ## '''
     162    ## XXX EAM : removed this function and made it more explicit (with an additional block)
     163    ## Returns a list of merged batch IDs that are merged but not yet deleted
     164    ## '''
     165    ## def getMergedButNotDeletedBatchIDs(self, batchType, column):
     166    ##
     167    ##     sql = "SELECT DISTINCT batch_id \
     168    ##            FROM batch \
     169    ##            WHERE timestamp > '" + self.skychunk.epoch + "' \
     170    ##            AND batch_type = '" + batchType + "' \
     171    ##            AND dvo_db = '" + self.skychunk.dvoLabel + "' \
     172    ##            AND merged = 1 \
     173    ##            AND " + column + " = 0"
     174    ##
     175    ##     ids = []
     176    ##     try:
     177    ##         rs = self.executeQuery(sql)
     178    ##         while (rs.next()): ids.append(rs.getInt(1))
     179    ##         rs.close()
     180    ##     except:
     181    ##         self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
     182    ##
     183    ##     self.logger.debug("Found %d merged but un-deleted items" % len(ids))
     184    ##
     185    ##     return ids
     186
     187    ## '''
     188    ## Returns a list of batch IDs marked as 'purged' but not yet deleted
     189    ## XXX EAM : removed this function and made it more explicit (with an additional block)
     190    ## '''
     191    ## def getPurgedButNotDeletedBatchIDs(self, batchType, column):
     192    ##
     193    ##     sql = "SELECT DISTINCT batch_id \
     194    ##            FROM batch \
     195    ##            WHERE timestamp > '" + self.skychunk.epoch + "' \
     196    ##            AND batch_type = '" + batchType + "' \
     197    ##            AND dvo_db = '" + self.skychunk.dvoLabel + "' \
     198    ##            AND purged = 1 \
     199    ##            AND " + column + " = 0"
     200    ##
     201    ##     ids = []
     202    ##     try:
     203    ##         rs = self.executeQuery(sql)
     204    ##         while (rs.next()): ids.append(rs.getInt(1))
     205    ##         rs.close()
     206    ##     except:
     207    ##         self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
     208    ##
     209    ##
     210    ##     self.logger.debug("Found %d merged but un-deleted items" % len(ids))
     211    ##
     212    ##     return ids
     213
     214    ## '''
     215    ## Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted
     216    ## '''
     217    ## def getLoadedToODMButNotDeletedBatchIDs(self, batchType, column):
     218    ##
     219    ##     sql = "SELECT DISTINCT batch_id \
     220    ##            FROM batch \
     221    ##            WHERE timestamp > '" + self.skychunk.epoch + "' \
     222    ##            AND batch_type = '" + batchType + "' \
     223    ##            AND dvo_db = '" + self.skychunk.dvoLabel + "' \
     224    ##            AND (loaded_to_ODM = -1 OR merge_worthy = 1) \
     225    ##            AND " + column + " = 0"
     226    ##
     227    ##     ids = []
     228    ##     try:
     229    ##         rs = self.executeQuery(sql)
     230    ##         while (rs.next()): ids.append(rs.getInt(1))
     231    ##         rs.close()
     232    ##     except:
     233    ##         self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
     234    ##
     235    ##     self.logger.debug("Found %d merged but un-deleted items" % len(ids))
     236    ##
     237    ##     return ids
     238
     239    # '''
     240    # Returns a list of merged batch IDs that are not deleted from local disk
     241    # '''
     242    # def getMergedButNotDeletedFromLocalDisk(self, batchType):
     243    #     return self.getMergedButNotDeletedBatchIDs(batchType, "deleted_local")
     244
     245    ## '''
     246    ## Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from local disk
     247    ## '''
     248    ## def getLoadedToODMButNotDeletedFromLocalDisk(self, batchType):
     249    ##     return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_local")
     250    ##
     251    ## '''
     252    ## Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from datastore
     253    ## '''
     254    ## def getLoadedToODMButNotDeletedFromDatastore(self, batchType):
     255    ##     return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_datastore")
     256    ##
     257    ## '''
     258    ## Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from DXLayer
     259    ## '''
     260    ## def getLoadedToODMButNotDeletedFromDXLayer(self, batchType):
     261    ##     return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_dxlayer")
     262
     263    ## '''
     264    ## Returns a list of purged  batch IDs that not deleted from local disk
     265    ## '''
     266    ## def getPurgedButNotDeletedFromLocalDisk(self, batchType):
     267    ##     return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_local")
     268    ##
     269    ## '''
     270    ## Returns a list of purged batch IDs that are not deleted from datastore
     271    ## '''
     272    ## def getPurgedButNotDeletedFromDatastore(self, batchType):
     273    ##     return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_datastore")
    142274
    143275

  • branches/eam_branches/ipp-20130307/ippTools/share

  • branches/eam_branches/ipp-20130307/ippTools/share/Makefile.am

    r35214 r35350  
    457457        laptool_WSdiff_check.sql \
    458458        laptool_stacks.sql \
     459        laptool_pendinggroup.sql \
     460        laptool_revertgroup.sql \
     461        laptool_filtersforgroup.sql \
    459462        vptool_find_rawexp.sql \
    460463        vptool_pendingimfile.sql \

  • branches/eam_branches/ipp-20130307/ippTools/share/addtool_find_processedexp_cam.sql

    r31375 r35350  
    11SELECT
    22    addProcessedExp.*,
    3     addRun.workdir
     3    addRun.label,
     4    addRun.workdir,
     5    camRun.cam_id,
     6    camRun.label as cam_label,
     7    camRun.data_group as cam_data_group,
     8    rawExp.exp_id,
     9    exp_name
    410FROM addProcessedExp
    511JOIN addRun

  • branches/eam_branches/ipp-20130307/ippTools/share/camtool_find_pendingimfile.sql

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippTools/share/chiptool_setimfiletoupdate.sql

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippTools/share/pxadmin_create_tables.sql

    r35239 r35350  
    14411441        outProduct VARCHAR(64),
    14421442        uri VARCHAR(255),
    1443         outdir     VARCHAR(255),
     1443        outdir VARCHAR(255),
     1444        username VARCHAR(255),
     1445        proj_id BIGINT,
     1446        registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    14441447        timestamp TIMESTAMP,
    14451448        fault SMALLINT,
    … …  
    14481451        KEY(state),
    14491452        KEY(fault),
    1450         KEY(label)
     1453        KEY(label),
     1454        KEY(username)
    14511455) ENGINE=innodb DEFAULT CHARSET=latin1;
    14521456
    … …  
    21942198    KEY (state),
    21952199    KEY (fault),
     2200    KEY (group_id),
    21962201    FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id),
    21972202    FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id),
    … …  
    22532258CREATE TABLE lapGroup  (
    22542259    seq_id  BIGINT,
     2260    tess_id VARCHAR(64),
    22552261    projection_cell VARCHAR(64),
    22562262    state   VARCHAR(16),
     2263    label   VARCHAR(64),
     2264    registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time group was registered
    22572265    fault   SMALLINT,
    2258     PRIMARY KEY(seq_id, projection_cell),
     2266    PRIMARY KEY(seq_id, tess_id, projection_cell),
    22592267    KEY(state),
    22602268    KEY(fault),

  • branches/eam_branches/ipp-20130307/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql

    r35213 r35350  
    11SELECT relExp.relexp_id
    2 FROM relGroup JOIN lapRun USING(lap_id)
     2FROM relGroup
     3    JOIN lapRun USING(lap_id)
     4    JOIN lapExp USING(lap_id)
    35    JOIN relExp using(rel_id, exp_id)
    46WHERE relExp.group_id = 0

  • branches/eam_branches/ipp-20130307/ippTools/share/releasetool_pendingrelgroup.sql

    r35214 r35350  
    1 SELECT relGroup.*
     1SELECT
     2    relGroup.*,
     3    ippRelease.release_name
    24FROM relGroup
    35    JOIN ippRelease USING(rel_id)

  • branches/eam_branches/ipp-20130307/ippTools/share/warptool_towarped.sql

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippTools/src

  • branches/eam_branches/ipp-20130307/ippTools/src/addtool.c

    r35228 r35350  
    876876        psTrace("addtool", PS_LOG_INFO, "no rows found");
    877877        psFree(output);
    878         return true;
     878        return false;
    879879    }
    880880

  • branches/eam_branches/ipp-20130307/ippTools/src/camtool.c

    r35239 r35350  
    599599        psTrace("camtool", PS_LOG_INFO, "no rows found");
    600600        psFree(output);
    601         return true;
     601        return false;
    602602    }
    603603

  • branches/eam_branches/ipp-20130307/ippTools/src/camtoolConfig.c

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippTools/src/laptool.c

    r34081 r35350  
    3434static bool inactiveexpMode(pxConfig *config);
    3535
     36// Groups
     37static bool definegroupMode(pxConfig *config);
     38static bool pendinggroupMode(pxConfig *config);
     39static bool filtersforgroupMode(pxConfig *config);
     40static bool updategroupMode(pxConfig *config);
     41static bool revertgroupMode(pxConfig *config);
     42static bool listgroupMode(pxConfig *config);
     43
    3644# define MODECASE(caseName, func) \
    3745  case caseName: \
    … …  
    6775   
    6876    MODECASE(LAPTOOL_MODE_INACTIVEEXP,   inactiveexpMode);
     77
     78    MODECASE(LAPTOOL_MODE_DEFINEGROUP,   definegroupMode);
     79    MODECASE(LAPTOOL_MODE_PENDINGGROUP,  pendinggroupMode);
     80    MODECASE(LAPTOOL_MODE_FILTERSFORGROUP, filtersforgroupMode);
     81    MODECASE(LAPTOOL_MODE_UPDATEGROUP,   updategroupMode);
     82    MODECASE(LAPTOOL_MODE_REVERTGROUP,   revertgroupMode);
     83    MODECASE(LAPTOOL_MODE_LISTGROUP,     listgroupMode);
     84
    6985  default:
    7086    psAbort("invalid option (this should not happen)");
    … …  
    900916  return(true);
    901917}
    902 
     918// ---------------------------
     919// Group level (a collection of completed lapRuns for a given lapSequence projection cell and collection of filters
     920
     921static bool definegroupMode(pxConfig *config)
     922{
     923  PS_ASSERT_PTR_NON_NULL(config, false);
     924
     925  PXOPT_LOOKUP_S64(seq_id,          config->args, "-seq_id",    true, false);
     926  PXOPT_LOOKUP_STR(label,           config->args, "-set_label", true, false);
     927
     928  PXOPT_LOOKUP_BOOL(pretend,        config->args, "-pretend",    false);
     929  PXOPT_LOOKUP_BOOL(simple,         config->args, "-simple",    false);
     930  PXOPT_LOOKUP_U64(limit,           config->args, "-limit",     false, false);
     931
     932  // the following insures that the -filter argument has been set up properly (adapted from pxAddLabelSearchArgs)
     933  psMetadataItem *item = psMetadataLookup(config->args, "-filter");
     934  psAssert (item, "-filter argument not found in config->args");
     935  psAssert (item->type == PS_DATA_METADATA_MULTI, "%s should be a multi container", "filter");
     936  psAssert (item->data.list->n, "%s should at least have a place-holder", "filter");
     937  psMetadataItem *entry = (psMetadataItem *)item->data.list->head->data;
     938  psAssert (entry, "%s should at least have a place-holder", "filter");
     939  // end of checking
     940
     941  // Now if the ony entry is the place-holder then the user supplied no -filter arguments
     942  // which are required
     943  if (!entry->data.str) {
     944    psError(PXTOOLS_ERR_ARGUMENTS, true, "at least one -filter is required");
     945    return false;
     946  }
     947
     948  int nFilters = item->data.list->n;
     949
     950  psString query = psStringCopy("SELECT seq_id, tess_id, projection_cell,\n");
     951
     952  // We construct a query joining completed lap runs with the same lapSequence.seq_id,
     953  // tess_id, and projection_cell
     954
     955  // select the lap_ids for the various filters. These are only used for debugging.
     956  psStringAppend(&query, "lap_id_%d", 0);
     957  for (int i = 1; i < nFilters; i++) {
     958    psStringAppend(&query, ", lap_id_%d", i);
     959  }
     960
     961
     962  // sub query for each supplied filter
     963  char * lapRunForFilter = "(\nSELECT seq_id, tess_id, projection_cell, lap_id as 'lap_id_%d'\n"
     964    "FROM lapRun\n"
     965    "WHERE filter LIKE '%s'\n"
     966    "   AND lapRun.seq_id = %"PRId64"\n"
     967    "   AND (lapRun.state = 'done' or lapRun.state = 'full')\n"
     968    " ) as lap_%d\n";
     969
     970  // loop over supplied filters and flesh out the query using the format above
     971  psListIterator *iter = psListIteratorAlloc (item->data.list, PS_LIST_HEAD, true);
     972  psMetadataItem *filterItem = NULL;
     973  int i = -1;
     974  while ((filterItem = psListGetAndIncrement(iter))) {
     975    ++i;
     976    if (i == 0) {
     977      psStringAppend(&query, "\nFROM\n");
     978    } else {
     979      psStringAppend(&query, "\nJOIN\n");
     980    }
     981
     982    psString filter = filterItem->data.str;
     983    psStringAppend(&query, lapRunForFilter, i, filter, seq_id, i);
     984
     985    if (i != 0) {
     986      psStringAppend(&query, "USING (seq_id, tess_id, projection_cell)\n");
     987    }
     988  }
     989  // now join to lapGroup
     990  psStringAppend(&query, "\nLEFT JOIN lapGroup USING(seq_id, tess_id, projection_cell)\n");
     991
     992  // we only want projection cells which do not already of an entry
     993  psStringAppend(&query, "\nWHERE lapGroup.projection_cell IS NULL\n");
     994  psStringAppend(&query, "\nORDER by projection_cell\n");
     995
     996  if (limit) {
     997    psString limitString = psDBGenerateLimitSQL(limit);
     998    psStringAppend(&query, "\n %s", limitString);
     999    psFree(limitString);
     1000  }
     1001
     1002  if (!p_psDBRunQuery(config->dbh, query)) {
     1003      psError(PS_ERR_UNKNOWN, false, "database error");
     1004      psFree(query);
     1005      return false;
     1006  }
     1007  psFree(query);
     1008
     1009  psArray *output = p_psDBFetchResult(config->dbh);
     1010  if (!output) {
     1011      psError(PS_ERR_UNKNOWN, false, "database error");
     1012      return false;
     1013  }
     1014
     1015  if (!psArrayLength(output)) {
     1016      psTrace("laptool", PS_LOG_INFO, "no rows found");
     1017      psFree(output);
     1018      return true;
     1019  }
     1020
     1021  if (pretend) {
     1022    if (!ippdbPrintMetadatas(stdout, output, "new_lapGroups", !simple)) {
     1023        psError(PS_ERR_UNKNOWN, false, "failed to print array");
     1024        psFree(output);
     1025        return false;
     1026    }
     1027    psFree(output);
     1028    return true;
     1029  }
     1030
     1031  if (!psDBTransaction(config->dbh)) {
     1032    psError(PS_ERR_UNKNOWN, false, "database error");
     1033    return false;
     1034  }
     1035
     1036  psTime *now = psTimeGetNow(PS_TIME_TAI);
     1037
     1038  psString last_projection_cell = NULL;
     1039  for (long i = 0; i < psArrayLength(output); i++) {
     1040    psMetadata *row = output->data[i];
     1041
     1042    psString tess_id = psMetadataLookupStr(NULL, row, "tess_id");
     1043    psString projection_cell = psMetadataLookupStr(NULL, row, "projection_cell");
     1044    if (last_projection_cell && !strcmp(last_projection_cell, projection_cell)) {
     1045        // duplicate lap runs for a filter will generate multiple rows
     1046        // Since we care about projection_cells not lapRuns per se this is not problem.
     1047        // Skip any duplicates.
     1048        continue;
     1049    }
     1050    last_projection_cell = projection_cell;
     1051
     1052    lapGroupRow *group = lapGroupRowAlloc(
     1053                                  seq_id,
     1054                                  tess_id,
     1055                                  projection_cell,
     1056                                  "new",  // state
     1057                                  label,
     1058                                  now,    // registered
     1059                                  0       // fault
     1060                                  );
     1061    if (!group) {
     1062      psError(PS_ERR_UNKNOWN, false, "failed to alloc lapGroup object");
     1063      psFree(output);
     1064      psFree(now);
     1065      return(false);
     1066    }
     1067
     1068
     1069    if (!lapGroupInsertObject(config->dbh, group)) {
     1070      if (!psDBRollback(config->dbh)) {
     1071        psError(PS_ERR_UNKNOWN, false, "database error");
     1072      }
     1073      psError(PS_ERR_UNKNOWN, false, "database error");
     1074      psFree(output);
     1075      return(true);
     1076    }
     1077  }
     1078
     1079  // point of no return
     1080  if (!psDBCommit(config->dbh)) {
     1081    psError(PS_ERR_UNKNOWN, false, "database error");
     1082    return false;
     1083  }
     1084
     1085  psFree(now);
     1086  psFree(output);
     1087
     1088  return(true); 
     1089}
     1090
     1091static bool pendinggroupMode(pxConfig *config)
     1092{
     1093  PS_ASSERT_PTR_NON_NULL(config, false);
     1094
     1095  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     1096  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",  false, false);
     1097 
     1098  psMetadata *where = psMetadataAlloc();
     1099  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
     1100  pxAddLabelSearchArgs(config, where, "-label", "lapRun.label", "==");
     1101 
     1102  psString query = pxDataGet("laptool_pendinggroup.sql");
     1103  if (!query) {
     1104    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
     1105    return false;
     1106  }
     1107
     1108  if (psListLength(where->list)) {
     1109    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     1110    psStringAppend(&query, " AND %s", whereClause);
     1111    psFree(whereClause);
     1112  }
     1113  if (limit) {
     1114    psString limitString = psDBGenerateLimitSQL(limit);
     1115    psStringAppend(&query, "\n %s", limitString);
     1116    psFree(limitString);
     1117  }
     1118 
     1119  if (!p_psDBRunQuery(config->dbh, query)) {
     1120    psError(PS_ERR_UNKNOWN, false, "database error");
     1121    psFree(query);
     1122    return false;
     1123  }
     1124  psFree(query);
     1125
     1126  psArray *output = p_psDBFetchResult(config->dbh);
     1127  if (!output) {
     1128    psErrorCode err = psErrorCodeLast();
     1129    switch (err) {
     1130    case PS_ERR_DB_CLIENT:
     1131      psError(PXTOOLS_ERR_SYS, false, "database error");
     1132    case PS_ERR_DB_SERVER:
     1133      psError(PXTOOLS_ERR_PROG, false, "database error");
     1134    default:
     1135      psError(PXTOOLS_ERR_PROG, false, "unknown error");
     1136    }
     1137
     1138    return false;
     1139  }
     1140  if (!psArrayLength(output)) {
     1141    psTrace("laptool", PS_LOG_INFO, "no rows found");
     1142    psFree(output);
     1143    return true;
     1144  }
     1145
     1146  if (psArrayLength(output)) {
     1147    if (!ippdbPrintMetadatas(stdout, output, "lapGroup", !simple)) {
     1148      psError(PS_ERR_UNKNOWN, false, "failed to print array");
     1149      psFree(output);
     1150      return false;
     1151    }
     1152  }
     1153
     1154  psFree(output);
     1155
     1156  return true;
     1157}
     1158static bool filtersforgroupMode(pxConfig *config)
     1159{
     1160  PS_ASSERT_PTR_NON_NULL(config, false);
     1161
     1162  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     1163 
     1164  psMetadata *where = psMetadataAlloc();
     1165  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
     1166  PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");
     1167  PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
     1168 
     1169  psString query = pxDataGet("laptool_filtersforgroup.sql");
     1170  if (!query) {
     1171    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
     1172    return false;
     1173  }
     1174
     1175  psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     1176  psStringAppend(&query, " WHERE %s", whereClause);
     1177  psFree(whereClause);
     1178
     1179  if (!p_psDBRunQuery(config->dbh, query)) {
     1180    psError(PS_ERR_UNKNOWN, false, "database error");
     1181    psFree(query);
     1182    return false;
     1183  }
     1184  psFree(query);
     1185
     1186  psArray *output = p_psDBFetchResult(config->dbh);
     1187  if (!output) {
     1188    psErrorCode err = psErrorCodeLast();
     1189    switch (err) {
     1190    case PS_ERR_DB_CLIENT:
     1191      psError(PXTOOLS_ERR_SYS, false, "database error");
     1192    case PS_ERR_DB_SERVER:
     1193      psError(PXTOOLS_ERR_PROG, false, "database error");
     1194    default:
     1195      psError(PXTOOLS_ERR_PROG, false, "unknown error");
     1196    }
     1197
     1198    return false;
     1199  }
     1200  if (!psArrayLength(output)) {
     1201    psTrace("laptool", PS_LOG_INFO, "no rows found");
     1202    psFree(output);
     1203    return true;
     1204  }
     1205
     1206  if (psArrayLength(output)) {
     1207    if (!ippdbPrintMetadatas(stdout, output, "filters", !simple)) {
     1208      psError(PS_ERR_UNKNOWN, false, "failed to print array");
     1209      psFree(output);
     1210      return false;
     1211    }
     1212  }
     1213
     1214  psFree(output);
     1215
     1216  return true;
     1217}
     1218
     1219static bool updategroupMode(pxConfig *config)
     1220{
     1221  PS_ASSERT_PTR_NON_NULL(config, false);
     1222
     1223  // check that all of the keys that define a lapGroup are supplied
     1224  PXOPT_LOOKUP_S64(seq_id, config->args, "-seq_id", true, false);
     1225  PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
     1226  PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false);
     1227
     1228  psMetadata *where = psMetadataAlloc();
     1229  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
     1230  PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");
     1231  PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
     1232
     1233  psMetadata *values = psMetadataAlloc();
     1234  PXOPT_COPY_STR(config->args, values, "-set_state", "state", "==");
     1235  PXOPT_COPY_STR(config->args, values, "-set_label", "label", "==");
     1236  PXOPT_COPY_S16(config->args, values, "-set_fault", "fault", "==");
     1237
     1238  if (!psListLength(values->list)) {
     1239    psFree(values);
     1240    psFree(where);
     1241    psError(PXTOOLS_ERR_ARGUMENTS, true, "must set at least one column");
     1242    return false;
     1243  }
     1244
     1245  long rows = psDBUpdateRows(config->dbh, "lapGroup", where, values);
     1246  if (rows < 1) {
     1247    psFree(values);
     1248    psError(PXTOOLS_ERR_SYS, true, "failed to update lapGroup for %" PRId64 " %s %s", seq_id, tess_id, projection_cell);
     1249    return false;
     1250  }
     1251  psFree(values);
     1252
     1253  return(true);
     1254}
     1255
     1256static bool revertgroupMode(pxConfig *config)
     1257{
     1258  PS_ASSERT_PTR_NON_NULL(config, false);
     1259
     1260  psMetadata *where = psMetadataAlloc();
     1261  pxAddLabelSearchArgs (config, where, "-label", "label", "==");
     1262  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
     1263  PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");
     1264  PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
     1265
     1266  if (!psListLength(where->list)) {
     1267    psFree(where);
     1268    psError(PXTOOLS_ERR_CONFIG, false, "search arguments are required");
     1269    return false;
     1270  }
     1271
     1272  psString query = pxDataGet("laptool_revertgroup.sql");
     1273  psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     1274  psStringAppend(&query, " AND %s", whereClause);
     1275  psFree(whereClause);
     1276  psFree(where);
     1277  if (!p_psDBRunQuery(config->dbh, query)) {
     1278    psError(PS_ERR_UNKNOWN, false, "database error");
     1279    psFree(query);
     1280    return false;
     1281  }
     1282  psFree(query);
     1283
     1284  psU64 affected = psDBAffectedRows(config->dbh);
     1285  psLogMsg("laptool", PS_LOG_INFO, "Updated %" PRIu64 " lapGroups", affected);
     1286
     1287  return true;
     1288}
     1289
     1290static bool listgroupMode(pxConfig *config)
     1291{
     1292  PS_ASSERT_PTR_NON_NULL(config, false);
     1293
     1294  psError(PXTOOLS_ERR_SYS, true, "not yet implemented");
     1295  return false;
     1296
     1297  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     1298  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",  false, false);
     1299 
     1300  psMetadata *where = psMetadataAlloc();
     1301  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
     1302  PXOPT_COPY_STR(config->args, where, "-seq_name",   "name",   "LIKE");
     1303
     1304  psString query = pxDataGet("laptool_listsequence.sql");
     1305  if (!query) {
     1306    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
     1307    return false;
     1308  }
     1309  if (psListLength(where->list)) {
     1310    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     1311    psStringAppend(&query, " WHERE %s", whereClause);
     1312    psFree(whereClause);
     1313  }
     1314
     1315  if (limit) {
     1316    psString limitString = psDBGenerateLimitSQL(limit);
     1317    psStringAppend(&query, " %s", limitString);
     1318    psFree(limitString);
     1319  }
     1320  if (!p_psDBRunQuery(config->dbh, query)) {
     1321    psError(PS_ERR_UNKNOWN, false, "database error");
     1322    psFree(query);
     1323    return false;
     1324  }
     1325  psFree(query);
     1326
     1327  psArray *output = p_psDBFetchResult(config->dbh);
     1328  if (!output) {
     1329    psErrorCode err = psErrorCodeLast();
     1330    switch (err) {
     1331    case PS_ERR_DB_CLIENT:
     1332      psError(PXTOOLS_ERR_SYS, false, "database error");
     1333    case PS_ERR_DB_SERVER:
     1334      psError(PXTOOLS_ERR_PROG, false, "database error");
     1335    default:
     1336      psError(PXTOOLS_ERR_PROG, false, "unknown error");
     1337    }
     1338
     1339    return false;
     1340  }
     1341  if (!psArrayLength(output)) {
     1342    psTrace("laptool", PS_LOG_INFO, "no rows found");
     1343    psFree(output);
     1344    return true;
     1345  }
     1346
     1347  if (psArrayLength(output)) {
     1348    if (!ippdbPrintMetadatas(stdout, output, "lapSequence", !simple)) {
     1349      psError(PS_ERR_UNKNOWN, false, "failed to print array");
     1350      psFree(output);
     1351      return false;
     1352    }
     1353  }
     1354
     1355  psFree(output);
     1356
     1357  return true;
     1358}

  • branches/eam_branches/ipp-20130307/ippTools/src/laptool.h

    r32422 r35350  
    2020  LAPTOOL_MODE_UPDATEEXP,
    2121  LAPTOOL_MODE_DIFFCHECK,
    22   LAPTOOL_MODE_INACTIVEEXP
     22  LAPTOOL_MODE_INACTIVEEXP,
     23  LAPTOOL_MODE_DEFINEGROUP,
     24  LAPTOOL_MODE_PENDINGGROUP,
     25  LAPTOOL_MODE_FILTERSFORGROUP,
     26  LAPTOOL_MODE_UPDATEGROUP,
     27  LAPTOOL_MODE_REVERTGROUP,
     28  LAPTOOL_MODE_LISTGROUP,
    2329} laptoolMode;
    2430

  • branches/eam_branches/ipp-20130307/ippTools/src/laptoolConfig.c

    r33343 r35350  
    9797  ADD_OPT(S64, updaterunArgs, "-set_quick_sass_id",           "set quick stack sass_id", 0);
    9898  ADD_OPT(S64, updaterunArgs, "-set_final_sass_id",           "set final stack sass_id", 0);
    99  
    10099 
    101100  // -pendingexp
    … …  
    150149  ADD_OPT(U64, inactiveexpArgs, "-limit",                     "limit result set to N items", 0);
    151150 
     151  // -definegroup
     152  psMetadata *definegroupArgs = psMetadataAlloc();
     153  ADD_OPT(S64, definegroupArgs, "-seq_id",                    "search by lap sequence ID (required)", 0);
     154  ADD_OPT(Str, definegroupArgs, "-set_label",                 "set label (required)", 0);
     155
     156  psMetadataAddStr(definegroupArgs,  PS_LIST_TAIL,
     157                                "-filter", PS_META_DUPLICATE_OK,
     158                                                              "search by filter (LIKE comparison, multiple OK)", NULL);
     159  ADD_OPT(Bool,definegroupArgs, "-pretend",                   "do not actuallym modify the database", false);
     160  ADD_OPT(Bool,definegroupArgs, "-simple",                    "use the simple output format", false);
     161  ADD_OPT(U64, definegroupArgs, "-limit",                     "limit result set to N items", 0);
     162
     163  // -pendinggroup
     164  psMetadata *pendinggroupArgs = psMetadataAlloc();
     165  psMetadataAddStr(pendinggroupArgs,  PS_LIST_TAIL,
     166                                "-label", PS_META_DUPLICATE_OK,
     167                                                              "search by label (multiple OK)", NULL);
     168  ADD_OPT(S64, pendinggroupArgs, "-seq_id",                    "search by lap sequence ID (required)", 0);
     169
     170  ADD_OPT(Bool,pendinggroupArgs, "-simple",                    "use the simple output format", false);
     171  ADD_OPT(U64, pendinggroupArgs, "-limit",                     "limit result set to N items", 0);
     172
     173  // -updategroup
     174  psMetadata *updategroupArgs = psMetadataAlloc();
     175  ADD_OPT(S64, updategroupArgs, "-seq_id",                    "search by lap sequence ID (required)", 0);
     176  ADD_OPT(Str, updategroupArgs, "-tess_id",                   "search by tess_id (required)", 0);
     177  ADD_OPT(Str, updategroupArgs, "-projection_cell",           "search by projection_cell (required)", 0);
     178  ADD_OPT(Str, updategroupArgs, "-set_state",                 "set state", 0);
     179  ADD_OPT(S16, updategroupArgs, "-set_fault",                 "set fault code", INT16_MAX);
     180  ADD_OPT(Str, updategroupArgs, "-set_label",                 "set label", 0);
     181
     182  // -revertgroup
     183  psMetadata *revertgroupArgs = psMetadataAlloc();
     184  psMetadataAddStr(revertgroupArgs,  PS_LIST_TAIL,
     185                                "-label", PS_META_DUPLICATE_OK,
     186                                                              "search by label (multiple OK)", NULL);
     187  ADD_OPT(S64, revertgroupArgs, "-seq_id",                    "search by lap sequence ID", 0);
     188  ADD_OPT(Str, revertgroupArgs, "-tess_id",                   "search by tess_id", 0);
     189  ADD_OPT(Str, revertgroupArgs, "-projection_cell",           "search by projection_cell", 0);
     190  ADD_OPT(S16, revertgroupArgs, "-fault",                     "fault code", INT16_MAX);
     191
     192  // -filtersforgroup
     193  psMetadata *filtersforgroupArgs = psMetadataAlloc();
     194  ADD_OPT(S64, filtersforgroupArgs, "-seq_id",                    "search by lap sequence ID (required)", 0);
     195  ADD_OPT(Str, filtersforgroupArgs, "-tess_id",                   "search by tess_id (required)", 0);
     196  ADD_OPT(Str, filtersforgroupArgs, "-projection_cell",           "search by projection_cell (required)", 0);
     197  ADD_OPT(Bool,filtersforgroupArgs, "-simple",                    "use the simple output format", false);
     198
    152199 
    153200  psMetadata *argSets = psMetadataAlloc();
    … …  
    166213  PXOPT_ADD_MODE("-diffcheck",               "", LAPTOOL_MODE_DIFFCHECK,        diffcheckArgs);
    167214  PXOPT_ADD_MODE("-inactiveexp",             "", LAPTOOL_MODE_INACTIVEEXP,      inactiveexpArgs);
     215  PXOPT_ADD_MODE("-definegroup",             "", LAPTOOL_MODE_DEFINEGROUP,      definegroupArgs);
     216  PXOPT_ADD_MODE("-pendinggroup",            "", LAPTOOL_MODE_PENDINGGROUP,     pendinggroupArgs);
     217  PXOPT_ADD_MODE("-filtersforgroup",         "", LAPTOOL_MODE_FILTERSFORGROUP,  filtersforgroupArgs);
     218  PXOPT_ADD_MODE("-updategroup",             "", LAPTOOL_MODE_UPDATEGROUP,      updategroupArgs);
     219  PXOPT_ADD_MODE("-revertgroup",             "", LAPTOOL_MODE_REVERTGROUP,      revertgroupArgs);
    168220 
    169221  if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

  • branches/eam_branches/ipp-20130307/ippTools/src/magictool.c

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippTools/src/pstamptool.c

    r34123 r35350  
    287287    PXOPT_LOOKUP_S64(ds_id,       config->args, "-ds_id", false, false);
    288288
     289    PXOPT_LOOKUP_STR(username,    config->args, "-username",  false, false);
     290    PXOPT_LOOKUP_S64(proj_id,     config->args, "-proj_id", false, false);
     291
     292    psTime *now = psTimeGetNow(PS_TIME_TAI);
     293
    289294    if (!pstampRequestInsert(config->dbh,
    290295        0,      // req_id
    … …  
    297302        uri,
    298303        NULL,   // outdir
    299         NULL,   // timestamp
     304        username,
     305        proj_id,
     306        now,    // registered
     307        now,    // timestamp
    300308        0       // fault
    301309        )) {
    … …  
    444452    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
    445453    PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!=");
    446     PXOPT_COPY_STR(config->args, where, "-name", "name", "==");
     454    PXOPT_COPY_STR(config->args, where, "-name", "name", "LIKE");
     455    PXOPT_COPY_STR(config->args, where, "-username", "username", "==");
     456    PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
    447457
    448458    PXOPT_LOOKUP_U64(limit,   config->args, "-limit",  false, false);
    … …  
    450460
    451461    if (!psListLength(where->list)) {
    452         psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied");
     462        psError(PS_ERR_UNKNOWN, true, "search paramters are required");
    453463        return false;
    454464    }
    … …  
    482492    if (!psArrayLength(output)) {
    483493        psTrace("pstamptool", PS_LOG_INFO, "request not found");
    484         // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for
     494        // This causes main to exit with PS_EXIT_DATA_ERROR which the pstamp scripts are looking for
    485495        psError(PXTOOLS_ERR_CONFIG, true, "request not found");
    486496        psFree(output);
    487         // we return false so that the caller can determine that a request does not exist
     497        // we return false so that the caller can easily determine that a request does not exist
    488498        return false;
    489499    }
    … …  
    567577    PXOPT_LOOKUP_STR(label,      config->args, "-set_label",      false, false);
    568578    PXOPT_LOOKUP_STR(outProduct, config->args, "-set_outProduct", false, false);
    569     PXOPT_LOOKUP_S16(fault,      config->args, "-set_fault",          false, false);
     579    PXOPT_LOOKUP_S16(fault,      config->args, "-set_fault",      false, false);
    570580    PXOPT_LOOKUP_STR(uri,        config->args, "-set_uri",        false, false);
    571     PXOPT_LOOKUP_STR(outdir,     config->args, "-set_outdir",        false, false);
     581    PXOPT_LOOKUP_STR(outdir,     config->args, "-set_outdir",     false, false);
    572582    PXOPT_LOOKUP_STR(name,       config->args, "-set_name",       false, false);
     583    PXOPT_LOOKUP_STR(username,   config->args, "-set_username",   false, false);
    573584    PXOPT_LOOKUP_STR(reqType,    config->args, "-set_reqType",    false, false);
    574     PXOPT_LOOKUP_BOOL(clearfault,config->args, "-clearfault",    false);
    575 
    576     if (!state && !label && !outProduct && !fault && !uri && !outdir && !name && !reqType && !clearfault) {
     585    PXOPT_LOOKUP_BOOL(clearfault,config->args, "-clearfault",     false);
     586
     587    if (!state && !label && !outProduct && !fault && !uri && !outdir && !name && !username && !reqType && !clearfault) {
    577588        psError(PS_ERR_UNKNOWN, true, "at least one set option is required");
    578589        return false;
    … …  
    584595    PXOPT_COPY_S32(config->args, where, "-fault",      "fault", "==");
    585596    PXOPT_COPY_STR(config->args, where, "-state",      "state", "==");
    586     PXOPT_COPY_STR(config->args, where, "-reqType",     "reqType", "==");
    587     PXOPT_COPY_STR(config->args, where, "-name",     "name", "LIKE");
     597    PXOPT_COPY_STR(config->args, where, "-reqType",    "reqType", "==");
     598    PXOPT_COPY_STR(config->args, where, "-name",       "name", "LIKE");
     599    PXOPT_COPY_STR(config->args, where, "-username",   "username", "==");
    588600    PXOPT_COPY_TIME(config->args, where, "-timestamp_begin", "timestamp", ">=");
    589601    PXOPT_COPY_TIME(config->args, where, "-timestamp_end", "timestamp", "<=");

  • branches/eam_branches/ipp-20130307/ippTools/src/pstamptoolConfig.c

    r33809 r35350  
    7474    psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0,    "define request file uri (required)", NULL);
    7575    psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0,  "define request ds_id", 0);
     76    psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-proj_id", 0, "define request proj_id", 0);
    7677    psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-name", 0,   "define request name", NULL);
     78    psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-username", 0, "define user name", NULL);
    7779    psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-label", 0,  "define request label", NULL);
    7880
    … …  
    8890    psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0,            "list by req_id", 0);
    8991    psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-name", 0,              "list by name", NULL);
     92    psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-username", 0,          "list by user name", NULL);
     93    psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-state", 0,              "list by state", NULL);
    9094    psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-not_req_id", 0,        "req_id to not list", 0);
    9195    psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    … …  
    105109    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0,        "search by state", NULL);
    106110    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0,      "search by reqType", NULL);
    107     psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0,      "search by reqType (LIKE comparsion)", NULL);
     111    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0,         "search by reqType (LIKE comparsion)", NULL);
     112    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-username", 0,     "search by username", NULL);
    108113    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL);
    109114    psMetadataAddTime(updatereqArgs, PS_LIST_TAIL, "-timestamp_begin", 0, "search by timestamp (>=)", NULL);
    … …  
    116121    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_uri", 0,          "new uri", NULL);
    117122    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_name", 0,         "new name", NULL);
     123    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_username", 0,     "new username", NULL);
    118124    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_reqType", 0,      "new reqType", NULL);
    119125    psMetadataAddBool(updatereqArgs, PS_LIST_TAIL, "-clearfault", 0,      "set fault to zero", false);

  • branches/eam_branches/ipp-20130307/ippTools/src/releasetool.c

    r35230 r35350  
    12701270        }
    12711271        if (!psArrayLength(exposures)) {
     1272            fprintf(stderr, "no exposures found for lap_id %" PRId64 "\n", lap_id);
     1273#ifdef notdef
     1274            continue;
    12721275            psFree(now);
    12731276            psFree(output);
    … …  
    12781281            }
    12791282            return false;
     1283#endif
    12801284        }
    12811285
    … …  
    13011305        psFree(exposures);
    13021306
    1303         // set state of relGroup to 'new'
     1307        // update the state of relGroup to 'new'
    13041308        if (!p_psDBRunQueryF(config->dbh, "UPDATE relGroup set state = 'new' WHERE group_id = %d", group_id)) {
    13051309            psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    13251329static bool updaterelgroupMode(pxConfig *config)
    13261330{
    1327     psError(PS_ERR_UNKNOWN, true, "not yet implemented");
    1328     return false;
     1331    PS_ASSERT_PTR_NON_NULL(config, false);
     1332
     1333    PXOPT_LOOKUP_S32(group_id, config->args,      "-group_id", true, false);
     1334
     1335    PXOPT_LOOKUP_STR(state,    config->args,    "-set_state",  false, false);
     1336    PXOPT_LOOKUP_STR(exp_list_path, config->args, "-set_exp_list_path", false, false);
     1337    PXOPT_LOOKUP_STR(new_label, config->args,   "-set_label", false, false);
     1338    PXOPT_LOOKUP_S16(fault,    config->args,    "-set_fault",  false, false);
     1339    PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault",  false);
     1340
     1341    if (!state && !exp_list_path && !new_label && !fault && !clearfault) {
     1342        psError(PXTOOLS_ERR_CONFIG, true, "must set at least one column\n");
     1343        return false;
     1344    }
     1345
     1346    if (fault && clearfault) {
     1347        psError(PXTOOLS_ERR_CONFIG, true, "cannot set and clear fault at same time\n");
     1348        return false;
     1349    }
     1350
     1351    psMetadata *where = psMetadataAlloc();
     1352
     1353    PXOPT_COPY_S64(config->args, where, "-group_id", "relGroup.group_id", "==");
     1354    // XXX TODO if label is enabled (for changing label or state for a block) we should
     1355    // disallow setting some parameters such as exp_list_path
     1356    //    PXOPT_COPY_S64(config->args, where, "-label", "relGroup.label", "==");
     1357    // make sure that we have enough parameters to identify the relGroup to change
     1358    if (!psListLength(where->list)){
     1359        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n");
     1360        psFree(where);
     1361        return false;
     1362    }
     1363
     1364    psString sep = "";
     1365    psString comma = ", ";
     1366    psString query = psStringCopy("UPDATE relGroup SET");
     1367    if (state) {
     1368        psStringAppend(&query, "%s relGroup.state = '%s'", sep, state);
     1369        sep = comma;
     1370    }
     1371    if (exp_list_path) {
     1372        psStringAppend(&query, "%s relGroup.exp_list_path = '%s'", sep, exp_list_path);
     1373        sep = comma;
     1374    }
     1375    if (new_label) {
     1376        psStringAppend(&query, "%s relGroup.label = '%s'", sep, new_label);
     1377        sep = comma;
     1378    }
     1379    if (fault) {
     1380        psStringAppend(&query, "%s relGroup.fault = %d", sep, fault);
     1381        sep = comma;
     1382    }
     1383    if (clearfault) {
     1384        psStringAppend(&query, "%s relGroup.fault = 0", sep);
     1385        sep = comma;
     1386    }
     1387
     1388    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     1389    psStringAppend(&query, " WHERE %s", whereClause);
     1390    psFree(whereClause);
     1391
     1392    if (!p_psDBRunQuery(config->dbh, query)) {
     1393        psError(PS_ERR_UNKNOWN, false, "database error");
     1394        psFree(query);
     1395        return false;
     1396    }
     1397
     1398    psU64 affected = psDBAffectedRows(config->dbh);
     1399    psLogMsg("releasetool", PS_LOG_INFO, "Updated %" PRIu64 " ippReleases", affected);
     1400
     1401
     1402    psFree(query);
     1403
     1404    return true;
    13291405}
    13301406

  • branches/eam_branches/ipp-20130307/ippTools/src/releasetoolConfig.c

    r35227 r35350  
    217217        // parameters of the relGroup
    218218    psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_type", 0,  "define group_type (required)", NULL);
    219     psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_name", 0,  "define group_name (required)", NULL);
     219    psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_name", 0,  "define group_name", NULL);
    220220    psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_label", 0, "define relgroup label (required)", NULL);
    221221    psMetadataAddStr(definerelgroupArgs,  PS_LIST_TAIL, "-set_state", 0,      "define state", NULL);
    … …  
    240240    psMetadata *updaterelgroupArgs = psMetadataAlloc();
    241241
    242         // set the target release
    243242    psMetadataAddS32(updaterelgroupArgs,  PS_LIST_TAIL, "-group_id", 0,   "select by group ID (required)", 0);
    244243

  • branches/eam_branches/ipp-20130307/ippconfig

  • branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppImage.config

    r35115 r35350  
    12321232  BIAS               BOOL    FALSE           # Bias subtraction
    12331233  DARK               BOOL    TRUE            # Dark subtraction
     1234  NOISEMAP           BOOL    FALSE
    12341235  REMNANCE           BOOL    FALSE           # Remnance masking
    12351236  SHUTTER            BOOL    FALSE           # Shutter correction

  • branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppStack.config

    r34125 r35350  
    5656END
    5757
     58STACK_ALLDEEP    METADATA                  # N>>20 inputs, typically MD fields, no cuts
     59    OUTPUT.NOCOMP           BOOL  TRUE
     60    OUTPUT.LOGFLUX          BOOL  FALSE
     61    OUTPUT.REPLICATE        BOOL  TRUE
     62END
     63
    5864# Recipe overrides for STACK
    5965STACKPHOT METADATA

  • branches/eam_branches/ipp-20130307/ippconfig/gpc1/psphot.config

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStack.config

    r34860 r35350  
    159159END
    160160
     161STACK_ALLDEEP   METADATA
     162END
     163
    161164TEST1 METADATA
    162165  PSF.AUTOSIZE    BOOL  FALSE           # Determine output PSF size from input PSFs?

  • branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStatsFromMetadata.config

    r35195 r35350  
    563563  ENTRY  VAL  ZPT_ERR             F32  CONSTANT         -zpt_stdev
    564564  # don't know why these come out as F64 instead of F32
    565   ENTRY  VAL  FWHM_MAJ            F64  CONSTANT         -fhwm_major
    566   ENTRY  VAL  FWHM_MIN            F64  CONSTANT         -fhwm_minor
     565  ENTRY  VAL  FWHM_MAJ            F64  CONSTANT         -fwhm_major
     566  ENTRY  VAL  FWHM_MIN            F64  CONSTANT         -fwhm_minor
    567567
    568568  ENTRY  VAL  QUALITY             S32  CONSTANT          -quality             # Bad quality flag

  • branches/eam_branches/ipp-20130307/ippconfig/recipes/ppSub.config

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippconfig/recipes/psphot.config

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/ippconfig/recipes/reductionClasses.mdc

    r35239 r35350  
    550550       STACK_PPSUB     STR      STACK
    551551       STACK_PSPHOT    STR      STACK
     552END
     553
     554# all/deep stacks
     555ALLDEEP_STACK            METADATA
     556        STACK_PPSTACK           STR     STACK_ALLDEEP
     557        STACK_PPSUB             STR     STACK
     558        STACK_PSPHOT            STR     STACK
     559        STACKPHOT_PSPHOT        STR     STACKPHOT
     560        STATICSKY_CALIBRATION   STR     STATICSKY_DEEPCAL
    552561END
    553562

  • branches/eam_branches/ipp-20130307/ppImage/src

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/psModules

  • branches/eam_branches/ipp-20130307/psModules/src/detrend/pmPattern.c

    r35081 r35350  
    997997      pmChip *chip = out->fpa->chips->data[i];
    998998      pmCell *cell = chip->cells->data[0]; // Cell of interest
    999       pmReadout *ro = cell->readouts->data[0]; // Readout of interest
    1000 
     999
     1000     
    10011001      psStatsInit(bgStats);
    10021002
    … …  
    10091009      int y = chipName[3] - '0';
    10101010
    1011      
     1011      if ((cell->readouts->n != 1)||(!chip->data_exists))  {
     1012        A->data.F64[y][x] = NAN;
     1013        B->data.F64[y][x] = NAN;
     1014        C->data.F64[y][x] = NAN;
     1015        D->data.F64[y][x] = NAN;
     1016        continue;
     1017      }
     1018      pmReadout *ro = cell->readouts->data[0]; // Readout of interest     
    10121019      for (int j = 0; j < 4; j++) {
    10131020        if (j == 0) {  // Region B
    … …  
    12841291        pmChip *chip = out->fpa->chips->data[i];
    12851292        pmCell *cell = chip->cells->data[0]; // Cell of interest
     1293        if ((cell->readouts->n != 1)||(!chip->data_exists))  {
     1294          continue;
     1295        }
    12861296        pmReadout *ro = cell->readouts->data[0]; // Readout of interest
    12871297        const char *chipName = psMetadataLookupStr(NULL,chip->concepts, "CHIP.NAME");

  • branches/eam_branches/ipp-20130307/psconfig

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/psphot

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/psphot/src

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/psphot/src/psphotStackImageLoop.c

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/pstamp/scripts

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/psvideophot

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20130307/tools/cleandsproduct.pl

    r31801 r35350  
    5252
    5353
    54 $datastore = "http://datastore.ipp.ifa.hawaii.edu" if !$datastore;
     54
     55# since this needs to run on the MHPCC cluster there is no need to go to the
     56# proxy to get the datastore listing
     57$datastore = "http://ippc17/ds" if !$datastore;
     58# $datastore = "http://datastore.ipp.ifa.hawaii.edu" if !$datastore;
    5559
    5660my $uri = "$datastore/$product/index.txt";

  • branches/eam_branches/ipp-20130307/tools/run.relastro

    r34870 r35350  
    66endif
    77
    8 set REALRUN = 0
     8set REALRUN = 1
    99
    1010set  catdir = $argv[1]
    … …  
    2121# -update-chips : determine improved astrometry calibrations
    2222# try 1000 meas per deg^2, but if memory if available, go to more
    23 if (1) then
     23if (0) then
    2424  set options = "-update-chips"
    2525  set options = "$options -D RELASTRO_SRC_MEAS_TOOFEW 1"
    … …  
    3434
    3535  # enable parallel operations for parallel databases
    36   # set options = "$options -parallel"
     36  set options = "$options -parallel"
    3737
    3838  set region = "-region $ra_min $ra_max $dec_min $dec_max"
    … …  
    5454
    5555  # enable parallel operations for parallel databases
    56   # set options = "$options -parallel"
     56  set options = "$options -parallel"
    5757
    5858  set region = "-region $ra_min $ra_max $dec_min $dec_max"

  • branches/eam_branches/ipp-20130307/tools/run.relphot

    r34875 r35350  
    4040
    4141  # activate parallel mode when catdir is parallel
    42   # set options = "$options -parallel"
     42  set options = "$options -parallel"
    4343
    4444  # option to change the schema

  • branches/eam_branches/ipp-20130307/tools/run.setphot

    r34870 r35350  
    66endif
    77
    8 set REALRUN = 0
     8set REALRUN = 1
    99
    1010set  catdir = $argv[1]
    … …  
    2222
    2323# activate parallel mode when catdir is parallel
    24 # set options = "$options -parallel"
     24set options = "$options -parallel"
    2525
    2626# the new ubercal file includes Eddie's offsets relative to JT (so, no need to supply -zpt-offsets)
Note: See TracChangeset for help on using the changeset viewer.