Index: /branches/eam_branches/ipp-20121219/ippToPsps/config/settings.xml
===================================================================
--- /branches/eam_branches/ipp-20121219/ippToPsps/config/settings.xml	(revision 34933)
+++ /branches/eam_branches/ipp-20121219/ippToPsps/config/settings.xml	(revision 34934)
@@ -5,5 +5,5 @@
 
   <!-- path for storing logs -->
-  <logPath>/data/ipp005.0/ipptopsps</logPath>
+  <logPath>/home/kiawe/eugene/ipptopsps</logPath>
 
   <!-- local scratch Db section -->
@@ -11,14 +11,14 @@
     <name>ipptopsps_scratch</name>
     <host>localhost</host>
-    <user>ipp</user>
-    <password>ipp</password>
+    <user>dvo</user>
+    <password>dvo</password>
   </localdatabase>
 
   <!-- GPC1 Db section -->
   <gpc1database>
-    <name>gpc1</name>
-    <host>ippdb01</host>
-    <user>ipp</user>
-    <password>ipp</password>
+    <name>gpc1test</name>
+    <host>localhost</host>
+    <user>dvo</user>
+    <password>dvo</password>
   </gpc1database>
 
@@ -26,7 +26,7 @@
   <ipptopspsdatabase>
     <name>ipptopsps</name>
-    <host>ippdb01</host>
-    <user>ipp</user>
-    <password>ipp</password>
+    <host>localhost</host>
+    <user>dvo</user>
+    <password>dvo</password>
   </ipptopspsdatabase>
 
Index: /branches/eam_branches/ipp-20121219/ippToPsps/doc/concerns.txt
===================================================================
--- /branches/eam_branches/ipp-20121219/ippToPsps/doc/concerns.txt	(revision 34934)
+++ /branches/eam_branches/ipp-20121219/ippToPsps/doc/concerns.txt	(revision 34934)
@@ -0,0 +1,28 @@
+
+I am listing here a number of concerns about the existing ippToPsps code
+
+1) the name ippToPsps is lame because both IPP and PSPS is in the
+   wrong and mismatched cases.  Alternative?  ipp2psps? ipp-psps?
+
+2) installation : there is none.  this must be fixed soon.  it is
+   completely unreasonable to have to cd to a specific directory to
+   run the software suite.
+
+3) gpc1 database : how will this work with multiple camera?
+
+4) czartool.  czartool code is completely irrelevant to ipp2psps and
+   should be moved.
+   
+
+5) gpc1 database selections : shouldn't these be folded into the
+   ippTools commands?  (or else changes will break this code)
+
+6) sql for gpc1 seems to use select distinct in places where it is not
+   correctly limiting the query (see, eg, getImageIDsForThisStackID)
+
+* need to create ipptopsps database with manual commands:
+* mysql> create database ipptopsps;
+pikake: mysql -h localhost -u dvo -p ipptopsps < config/ippToPspsDbSchema.sql
+
+* need to copy / install jython tarball somewhere
+
Index: /branches/eam_branches/ipp-20121219/ippToPsps/jython/gpc1db.py
===================================================================
--- /branches/eam_branches/ipp-20121219/ippToPsps/jython/gpc1db.py	(revision 34933)
+++ /branches/eam_branches/ipp-20121219/ippToPsps/jython/gpc1db.py	(revision 34934)
@@ -112,4 +112,5 @@
         self.logger.debug("Querying GPC1 for image IDs for stack ID: " + str(stackID))
 
+        # JOIN stackInputSkyfile shoule use stack_id as well as warp_id, right? (maybe not -- the join to warp is first)
         sql = "SELECT DISTINCT CONCAT(exp_id, SUBSTR(class_id, 3, 4)) FROM ( \
                SELECT DISTINCT exp_id,class_id \
Index: /branches/eam_branches/ipp-20121219/ippToPsps/test/mkgpc1.sh
===================================================================
--- /branches/eam_branches/ipp-20121219/ippToPsps/test/mkgpc1.sh	(revision 34933)
+++ /branches/eam_branches/ipp-20121219/ippToPsps/test/mkgpc1.sh	(revision 34934)
@@ -18,59 +18,29 @@
 
 if ("$1" == "create") then
-  if ($#argv != 4) goto usage;
+  if ($#argv < 4) goto usage;
   set dbhost = $2
   set dbname = $3
   set dbuser = $4
+  if ($#argv == 5) then
+    set dbpass = "-p$5"
+  else 
+    set dbpass = "-p"
+  endif
 
   # create a database with completely minimal tables needed to test
-  mysql -h $dbhost -u $dbuser -p <<EOF
+  mysql -h $dbhost -u $dbuser $dbpass <<EOF
    CREATE DATABASE $dbname;
    USE $dbname;
-  CREATE TABLE addRun (
-    stage CHAR 64,
-    state CHAR 64,
-    stage_id INT,
-    minidvodb_name CHAR 64
+  CREATE TABLE ippToPspsFake (
+    my_id INT
   );
-  CREATE TABLE camProcessedExp (
-    path_base CHAR 255,
-    cam_id INT
-  );
-  CREATE TABLE camRun (
-    cam_id INT,
-    chip_id INT,
-    dist_group CHAR 64,
-    software_ver CHAR 16
-  );
-  CREATE TABLE chipRun (
-    exp_id INT,
-    chip_id INT
-  );
-  CREATE TABLE fakeRun (
-    fake_id INT,
-    cam_id INT
-  );
-  CREATE TABLE mergedvodbProcessed (
-   merge_id INT,
-   fault INT
-  );
-  CREATE TABLE mergedvodbRun (
-   minidvodb_id INT,
-   merge_id INT,
-   state CHAR 64,
-   mergedvodb CHAR 64
-  );
-  CREATE TABLE minidvodbProcessed (
-   minidvodb_id INT,
-   fault INT
-  );
-  CREATE TABLE minidvodbRun (
-   minidvodb_name CHAR 64,
-   minidvodb_id INT,
-   state CHAR 64
+  CREATE TABLE skycell (
+   radeg REAL,
+   decdeg REAL,
+   skycell_id CHAR(64)
   );
   CREATE TABLE rawExp (
    exp_id INT,
-   exp_name CHAR 64,
+   exp_name CHAR(64),
    exp_time REAL,
    bg REAL,
@@ -79,7 +49,43 @@
    decl FLOAT
   );
-  CREATE TABLE skycalResult (
-   path_base CHAR 255,
-   skycal_id INT
+  CREATE TABLE chipRun (
+    exp_id INT,
+    chip_id INT
+  );
+  CREATE TABLE camRun (
+    chip_id INT,
+    cam_id INT,
+    dist_group CHAR(64),
+    software_ver CHAR(16)
+  );
+  CREATE TABLE camProcessedExp (
+    path_base CHAR(255),
+    cam_id INT
+  );
+  CREATE TABLE fakeRun (
+    fake_id INT,
+    cam_id INT
+  );
+  CREATE TABLE warpRun (
+   warp_id INT,
+   fake_id INT
+  );
+  CREATE TABLE warpSkyCellMap (
+   warp_id INT,
+   skycell_id CHAR(64),
+   class_id CHAR(64)
+  );
+  CREATE TABLE stackRun (
+   stack_id INT,
+   skycell_id CHAR(64),
+   filter CHAR(64),
+   software_ver CHAR(16)
+  );
+  CREATE TABLE stackInputSkyfile (
+   stack_id INT,
+   warp_id INT
+  );
+  CREATE TABLE stackSumSkyfile (
+   stack_id INT
   );
   CREATE TABLE skycalRun (
@@ -87,28 +93,32 @@
    stack_id INT
   );
-  CREATE TABLE skycell (
-   radec REAL,
-   decdeg REAL,
-   skycell_id CHAR 64
+  CREATE TABLE skycalResult (
+   path_base CHAR(255),
+   skycal_id INT
   );
-  CREATE TABLE stackInputSkyfile (
-   warp_id INT
+  CREATE TABLE addRun (
+    stage CHAR(64),
+    state CHAR(64),
+    stage_id INT,
+    minidvodb_name CHAR(64)
   );
-  CREATE TABLE stackRun (
-   stack_id INT,
-   skycell_id CHAR 64,
-   filter CHAR 64,
-   software_ver CHAR 16
+  CREATE TABLE minidvodbRun (
+   minidvodb_name CHAR(64),
+   minidvodb_id INT,
+   state CHAR(64)
   );
-  CREATE TABLE stackSumSkyfile (
-   stack_id INT
+  CREATE TABLE minidvodbProcessed (
+   minidvodb_id INT,
+   fault INT
   );
-  CREATE TABLE warpRun (
-   warp_id INT
+  CREATE TABLE mergedvodbRun (
+   minidvodb_id INT,
+   merge_id INT,
+   state CHAR(64),
+   mergedvodb CHAR(64)
   );
-  CREATE TABLE warpSkyCellMap (
-   warp_id INT,
-   skycell_id CHAR 64,
-   class_id CHAR 64
+  CREATE TABLE mergedvodbProcessed (
+   merge_id INT,
+   fault INT
   );
 EOF
@@ -117,21 +127,32 @@
 
 if ("$1" == "delete") then
-  if ($#argv != 4) goto usage;
+  if ($#argv < 4) goto usage;
   set dbhost = $2
   set dbname = $3
   set dbuser = $4
+  if ($#argv == 5) then
+    set dbpass = "-p$5"
+  else 
+    set dbpass = "-p"
+  endif
 
-  mysql -h $dbhost -u $dbuser -p <<EOF > /dev/null
+  mysql -h $dbhost -u $dbuser $dbpass <<EOF
    USE $dbname;
-   describe dvoDetection;
+   describe ippToPspsFake;
 EOF
-
   if ($status) then
-    echo "database does not contain dvoDetections, not deleting"
+    echo "database does not contain ippToPspsFake, not deleting"
     exit 1;
   endif
 
+  set ntable = `echo "USE $dbname; show tables;" | mysql -h $dbhost -u $dbuser $dbpass | wc -l`
+  echo ntable: $ntable
+  if ($ntable != 20) then
+    echo "expected 20 lines (19 tables), but got $ntable, not deleting"
+    exit 2
+  endif
+
   echo "database is valid, deleting"
-  mysql -h $dbhost -u $dbuser -p <<EOF
+  mysql -h $dbhost -u $dbuser $dbpass <<EOF
    drop database $dbname;
 EOF
@@ -141,13 +162,13 @@
 
 usage:
-  echo "USAGE: dbadmin.sh (options)"
+  echo "USAGE: mkgpc1.sh (options)"
   echo ""
-  echo "  dbadmin.sh create (dbhost) (dbname) (dbuser)"
+  echo "  mkgpc1.sh create (dbhost) (dbname) (dbuser) [dbpass]"
   echo "      create a new test db"
   echo ""
-  echo "  dbadmin.sh delete (dbhost) (dbname) (dbuser)"
-  echo "      delete an existing test db (enter pass 2x)"
+  echo "  mkgpc1.sh delete (dbhost) (dbname) (dbuser) [dbpass]"
+  echo "      delete an existing test db (enter pass 3x)"
   echo ""
-  echo "  dbadmin.sh user (dbhost) (dbuser) (password)"
+  echo "  mkgpc1.sh user (dbhost) (dbuser) (password)"
   echo "      create a new user and password for the test db"
   echo ""
Index: /branches/eam_branches/ipp-20121219/ippToPsps/test/mkgpc1data.dvo
===================================================================
--- /branches/eam_branches/ipp-20121219/ippToPsps/test/mkgpc1data.dvo	(revision 34934)
+++ /branches/eam_branches/ipp-20121219/ippToPsps/test/mkgpc1data.dvo	(revision 34934)
@@ -0,0 +1,226 @@
+# -*-sh-*-
+input tap.dvo
+
+if (not($?VERBOSE)) set VERBOSE = 0
+
+macro test.all
+  test.relphot PS1_V2 	PS1_V2
+end  
+
+# imagedata describes the full fake exposure set.  the fields are
+# exp_id stack_id raw_file exp_file offset
+list imagedata
+ 1 1 test.01.txt test.01.cmf  0.000 
+ 2 1 test.02.txt test.02.cmf -0.025
+ 3 1 test.03.txt test.03.cmf  0.025
+ 4 1 test.04.txt test.04.cmf  0.010
+ 5 2 test.05.txt test.05.cmf  0.000 
+ 6 2 test.06.txt test.06.cmf -0.025
+ 7 2 test.07.txt test.07.cmf  0.025
+ 8 2 test.08.txt test.08.cmf  0.010
+ 9 3 test.09.txt test.09.cmf  0.000 
+10 3 test.10.txt test.10.cmf -0.025
+11 3 test.11.txt test.11.cmf  0.025
+12 3 test.12.txt test.12.cmf  0.010
+end
+
+list stackdata
+ 1 stack.01.txt stack.01.cmf 0.010
+ 2 stack.02.txt stack.02.cmf 0.020
+ 3 stack.03.txt stack.03.cmf 0.030
+end
+ 
+macro init.db
+
+  $dbhost = localhost
+  $dbname = gpc1test
+  $dbuser = dvo
+  $dbpass = dvo
+
+  $RA = 10.0
+  $DEC = 20.0
+
+  exec mkgpc1.sh delete $dbhost $dbname $dbuser $dbpass
+  exec mkgpc1.sh create $dbhost $dbname $dbuser $dbpass
+
+  dbconnect $dbhost $dbuser $dbname
+
+  # system-wide info
+  dbinsert ippToPspsFake (my_id) values (2)
+  dbinsert skycell (radeg, decdeg, skycell_id) values ($RA, $DEC, "skycell.0000.000")
+end
+
+# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp
+macro mkcatdir.cam
+  if ($0 != 3)
+    echo "mkcatdir.cam (cmftype) (dvotype)"
+    break
+  end
+
+  # tapPLAN 4
+
+  local i catdir RA DEC ID rawfile cmffile offset
+  $catdir = catdir.cam
+  exec rm -rf $catdir
+
+  $RA = 10.0
+  $DEC = 20.0
+
+  for i 0 $imagedata:n
+    list word -split $imagedata:$i
+    $ID = $word:0
+    $rawfile = $word:2
+    $cmffile = $word:3
+    $offset = $word:4
+
+    mkinput $offset $rawfile
+    exec mkcmf $rawfile $cmffile -date 2008/1/1 -time $i\:00:00 -radec $RA $DEC -type $1
+    exec addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
+  end
+
+  # for i 0 $offset:n
+  #   tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i"
+  # end
+
+  # tapDONE
+end
+
+# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp
+macro mkcatdir.stk
+  if ($0 != 3)
+    echo "mkcatdir.stk (cmftype) (dvotype)"
+    break
+  end
+
+  # tapPLAN 4
+
+  local i catdir RA DEC ID rawfile cmffile offset
+  $catdir = catdir.stk
+  exec rm -rf $catdir
+
+  $RA = 10.0
+  $DEC = 20.0
+
+  for i 0 $stackdata:n
+    list word -split $stackdata:$i
+    $ID = $word:0
+    $rawfile = $word:1
+    $cmffile = $word:2
+    $offset = $word:3
+
+    mkinput $offset $rawfile
+    exec mkcmf $rawfile $cmffile -date 2008/1/1 -time $i\:00:00 -radec $RA $DEC -type $1
+    exec addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
+  end
+
+  # for i 0 $offset:n
+  #   tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i"
+  # end
+
+  # tapDONE
+end
+
+macro insert.exp
+  if ($0 != 3)
+    echo "USAGE: insert.exp (ID) (filename)"
+    break
+  end
+
+  local myID filename
+  $myID = $1
+  $filename = $2
+
+  # a single exposure
+  dbinsert rawExp (exp_id, exp_name, exp_time, bg, bg_stdev, ra, decl) values ($myID, "test.$myID", 10.0, 100.0, 2.0, $RA, $DEC)
+  dbinsert chipRun (exp_id, chip_id) values ($myID, $myID)
+  dbinsert camRun (chip_id, cam_id, dist_group, software_ver) values ($myID, $myID, "testgroup", "testsoft")
+  dbinsert camProcessedExp (cam_id, path_base) values ($myID, "$filename")
+  dbinsert fakeRun (fake_id, cam_id) values ($myID, $myID)
+  dbinsert warpRun (warp_id, fake_id) values ($myID, $myID)
+  dbinsert warpSkyCellMap (warp_id, skycell_id, class_id) values ($myID, "skycell.0000.000", "XY00")
+
+  dbinsert addRun (stage, state, stage_id, minidvodb_name) values ("cam", "full", $myID, "catdir.cam")
+end
+
+# this inserts a single stack, adding the exposures associated with this stack
+macro insert.stack
+  if ($0 != 3)
+    echo "USAGE: insert.stack (ID) (stkfile)"
+    break
+  end
+
+  local i stkID expID expfile stkfile
+  
+  $stkID = $1
+  $stkfile = $2
+
+  dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "skycell.0000.000", "g.00000", "testsoft")
+
+  for i 0 $imagedata:n
+    echo $imagedata:$i
+    list tmp -split $imagedata:$i
+    if ($stkID != $tmp:1) continue
+
+    $expID = $tmp:0
+    $expfile = $tmp:3
+    insert.exp $expID $expfile
+    dbinsert stackInputSkyfile (stack_id, warp_id) values ($stkID, $expID)
+  end
+  dbinsert stackSumSkyfile (stack_id) values ($stkID)
+  dbinsert skycalRun (skycal_id, stack_id) values ($stkID, $stkID)
+  dbinsert skycalResult (skycal_id, path_base) values ($stkID, "$stkfile")
+  
+  dbinsert addRun (stage, state, stage_id, minidvodb_name) values ("skycal", "full", $stkID, "catdir.stk")
+end
+
+
+macro insert.stack.set
+
+  local i
+
+  for i 0 $stackdata:n
+    list word -split $stackdata:$i
+    $ID = $word:0
+    $cmffile = $word:2
+    insert.stack $ID $cmffile
+  end
+
+  # I am setting these databases in a very fake way, with same in and out names!
+
+  # catdirs for the exposures
+  dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.cam", 1, "merged")
+  dbinsert minidvodbProcessed (minidvodb_id, fault) values (1, 0)
+  
+  dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (1, 1, "merged", "catdir.cam")
+  dbinsert mergedvodbProcessed (merge_id, fault) values (1, 0)
+
+  # catdirs for the stack
+  dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.stk", 2, "merged")
+  dbinsert minidvodbProcessed (minidvodb_id, fault) values (2, 0)
+  
+  dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (2, 2, "merged", "catdir.stk")
+  dbinsert mergedvodbProcessed (merge_id, fault) values (2, 0)
+end
+
+
+# make a simple input file for mkcmf
+macro mkinput
+  if ($0 != 3)
+    echo "mkinput (offset) (output)"
+    break
+  end
+
+  local output 
+  $output = $1
+
+  exec rm -f $output
+
+  local i j
+  output $output
+  for i 10 1024 100
+    for j 10 1024 100
+      fprintf " %6.1f %6.1f  %7.3f" $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0}
+    end
+  end
+  output stdout
+end
Index: /branches/eam_branches/ipp-20121219/ippToPsps/test/tap.dvo
===================================================================
--- /branches/eam_branches/ipp-20121219/ippToPsps/test/tap.dvo	(revision 34934)
+++ /branches/eam_branches/ipp-20121219/ippToPsps/test/tap.dvo	(revision 34934)
@@ -0,0 +1,72 @@
+# -*-sh-*-
+
+if (not($?TAP_BREAK)) set TAP_BREAK = 0
+if (not($?TAP_VERBOSE)) set TAP_VERBOSE = 0
+
+macro tapOK
+  if ($0 != 3) 
+    echo "USAGE: tapOK (condition) (message)"
+    break
+  end
+
+  if ($1)
+    if ($TAP_VERBOSE)
+      echo "ok : $2"
+    end
+    $TAP_LAST = 1
+  else
+    echo "not ok : $2"
+    $TAP_NFAIL ++
+    $TAP_LAST = 0
+    if ($TAP_BREAK)
+     break
+    end
+  end
+  $TAP_NDONE ++
+end
+
+macro tapPLAN
+  if ($0 != 2) 
+    echo "USAGE: tapPLAN (Ntests)"
+    break
+  end
+
+  $TAP_NTEST = $1
+  $TAP_NFAIL = 0
+  $TAP_NSKIP = 0
+  $TAP_NDONE = 0
+  if (not($?TAP_BREAK)) set TAP_BREAK = 0
+end
+
+macro tapSKIP
+  if ($0 != 2) 
+    echo "USAGE: tapSKIP (nskip)"
+    break
+  end
+
+  $TAP_NSKIP += $1
+  $TAP_NDONE += $1
+end
+
+macro tapDONE
+  if ($0 != 1) 
+    echo "USAGE: tapDONE"
+    break
+  end
+
+  if ($TAP_NDONE != $TAP_NTEST) 
+    echo "planned tests ($TAP_NTEST) not equal to done tests ($TAP_NDONE)"
+  end
+
+  if ($TAP_NFAIL) 
+    echo "failed $TAP_NFAIL of $TAP_NDONE"
+  end
+
+  if ($TAP_NSKIP) 
+    echo "skipped $TAP_NSKIP of $TAP_NDONE"
+  end
+
+  if ($TAP_NFAIL + $TAP_NSKIP == 0) 
+    echo "passed $TAP_NDONE tests"
+  end
+end