#!/usr/bin/env jython

#
# The main loading program for ippToPsps. See usage below.
#

import time
import sys
import os
import math
import traceback
import logging.config

from ipptopsps import IppToPsps
from scratchdb import ScratchDb
from gpc1db import Gpc1Db
from datastore import Datastore
from batch import Batch
from dvo import Dvo

from initbatch import InitBatch
from stackbatch import StackBatch
from detectionbatch import DetectionBatch
from objectbatch import ObjectBatch

from diffbatch import DiffBatch 
from diffobjectbatch import DiffObjectBatch

from forcedwarpbatch import ForcedWarpBatch 
from forcedobjectbatch import ForcedObjectBatch
from forcedgalaxybatch import ForcedGalaxyBatch
#
#
'''
Loader class
'''
class Loader(IppToPsps):

    '''
    Constructor
    '''
    def __init__(self, argv):
        super(Loader, self).__init__(argv, 1, 1)

        if self.skychunk.parallel:
            self.logger.info("using PARALLEL DVO")
        else: 
            self.logger.info("using SERIAL DVO")

        # connect to the gpc1 database
        self.gpc1Db = Gpc1Db(self.logger, self.config)

        # connect to scratch database
        self.scratchDb = ScratchDb(self.logger, self.config)
        if not self.config.test and self.scratchDb.anyOtherConnections():
            self.logger.errorPair("This scratch Db is already in use", self.scratchDb.dbName)
            self.scratchDb.disconnect()
            self.exitProgram("Cannot connect to a scratch database")
            raise
       
        # create Datastore objects
        self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb)

        # if an IN batch is requested, create and quit
        if len(sys.argv) > 2 and sys.argv[2] == "init":
           batchID = self.ippToPspsDb.createNewBatch("IN", 0)
           if batchID > 0:
               batch = InitBatch(self.logger, 
                       self.config, 
                       self.skychunk, 
                       self.gpc1Db, 
                       self.ippToPspsDb, 
                       self.scratchDb, 
                       batchID)
               batch.run()
           else:
               self.logger.errorPair("Could not create batch ID", "%d" % batchID)
               self.exitProgram("failed to create init batch")
               
           self.exitProgram("init batch created")
    

        # if the 'once' option is passed, we do not loop multiple times in 'run'
        self.onePassOnly = 0
        if len(sys.argv) > 2 and sys.argv[2] == "once":
           self.onePassOnly = 1

        if self.config.loopBatch:
            self.parsePollTimeArg("0.0166")
        else:
            self.parsePollTimeArg("0.0")

        self.skychunk.printAll()

    '''
    Overrides base-class version so we can ensure our scratch Db is using the right DVO Db
    '''
    def refreshSkychunk(self):

        ret = super(Loader, self).refreshSkychunk()
        try: self.scratchDb
        except: return ret
           
        self.dvo = Dvo(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb, self.gpc1Db)

        return ret

    '''
    Overrides base_class version so we can break out of processing loops if the skychunk changes
    '''
    def checkClientStatus(self):

        oldSkychunkName = self.skychunk.name
        super(Loader, self).checkClientStatus()
        if oldSkychunkName != self.skychunk.name:
            self.logger.infoPair("Skychunk changed",  "from '" + oldSkychunkName + "' to '" + self.skychunk.name + "'")
            self.skychunk.printAll()
            return False

        return True

    '''
    Main processing loop
    '''
    def run(self):

        if not self.scratchDb.connected: return

        numAttempts = 0
        abort = False
        while True:

            # get a stripe's worth of box IDs
            boxIds = self.ippToPspsDb.getStripeBoxIdsAnyBatch(self.HOST, self.PID)
            self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds))
            self.logger.infoPair("numAttempts", str(numAttempts))

            if len(boxIds) > 0: numAttempts = 0
            numAttempts += 1

            for boxId in boxIds:

                # get box info. if boxes have changed, break and start again
                try:
                    boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
                except:
                    self.logger.infoPair("Can't get details for this box", "%d" % boxId)
                    break
                self.logger.infoPair("client status", "ok")

                for batchType in self.skychunk.batchTypes:

                    self.logger.infoPair("Check","client status")
                    if not self.checkClientStatus():
                        abort = True
                        break
                    self.logger.infoPair("client status", "ok")

                    ids = self.ippToPspsDb.getPendingIdsForThisBox(boxId, batchType)
               
                    # ids are the stage_ids for items to be processed (eg, stack_id for a stack CMF, cam_id for P2 smf)

                    if len(ids) < 1: 
                        self.logger.debugPair("No " + batchType + " items found in this box", "skipping")
                        continue
                
                    self.logger.infoSeparator()
                    self.logger.infoTitle("New box")
                    self.logger.infoPair("Box dimensions: ", "ra / dec / size : %.1f / %.1f / %.1f   --   sky bounds : (%.2f - %2.f, %2.f - %2.f)" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE'], boxDim['minRA'], boxDim['maxRA'], boxDim['minDEC'], boxDim['maxDEC']))
                    self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids))
                    #previously said if not OB, but we don't need these for Do or FO
                    if (batchType == "P2" or batchType == "ST" or batchType =="DF" or batchType == "FW"):
                        if self.config.resetDvo or not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost):
                            if not self.dvo.nativeIngestDetections(boxId, boxDim, batchType):
                                self.logger.error("Unable to ingest detections with nativeIngest")
                                return False

                    '''
                    NOTE EAM : "TheseItems" refers to each of the stack CMFs, camera SMFs, or object batches
                    '''
                    if not self.processTheseItems(batchType, ids):
                        abort = True
                        break
                    self.logger.infoPair("processed","ok")

            self.logger.infoPair("done with processing boxes, numAttempts: ", str(numAttempts))
            if abort or not self.checkClientStatus(): abort = True
            elif numAttempts > 1 and not self.waitForPollTime():  break

            if self.onePassOnly: self.exitProgram("one pass completed")
        
    '''
    Actually loops through items and processes them
    '''
    def processTheseItems(self, batchType, ids):
    
        self.logger.infoPair("%s items queued" % batchType, "%d" % len(ids))
    
        # loop round IDs of all items to be processed
        self.ippToPspsDb.lockBatchTable()
        self.logger.infoPair("heather:","lockbatchtable")
        unattemptedCount = 0
        for id in ids:
            self.logger.infoPair("heather:","in ids")
            #self.logger.infoPair("heather:id",str(id))
            batchID = self.ippToPspsDb.createNewBatch(batchType, id)
            #self.logger.infoPair("heather:","creatednewbatch")
            #self.logger.infoPair("heather:batchId", str(batchID))
            if batchID < 0: 
                unattemptedCount += 1
                continue
            self.logger.infoPair("heather:","passed logic")

            self.ippToPspsDb.unlockTables()
            #self.logger.infoPair("heather:","unlock table")

            self.ippToPspsDb.deletePendingItem(batchType, id)
            #self.logger.infoPair("heather:","deleted pending item")

            # catch any raised exceptions in batch constructors
            try:
                if batchType == "P2":
                    self.logger.infoPair("defining P2 batch for cam_id:",id)
                    batch = DetectionBatch(self.logger,
                            self.config,
                            self.skychunk,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID)
                elif batchType == "ST":
                    self.logger.infoPair("defining ST batch for sky_id:",id)
                    batch = StackBatch(self.logger,
                            self.config,
                            self.skychunk,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID)
    
                elif batchType == "OB":
                    batch = ObjectBatch(self.logger,
                            self.config,
                            self.skychunk,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID)
    
                elif batchType == "DF":
                    self.logger.infoPair("wegothere!","we are awesome")
                    batch = DiffBatch(self.logger,
                            self.config,
                            self.skychunk,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID)
    
                elif batchType == "DO":
                    batch = DiffObjectBatch(self.logger,
                            self.config,
                            self.skychunk,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID)
    
                elif batchType == "FW":
                    batch = ForcedWarpBatch(self.logger,
                            self.config,
                            self.skychunk,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID)
    
                elif batchType == "FO":
                    batch = ForcedObjectBatch(self.logger,
                            self.config,
                            self.skychunk,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID)

                elif batchType == "FG":
                    batch = ForcedGalaxyBatch(self.logger,
                                              self.config,
                                              self.skychunk,
                                              self.gpc1Db,
                                              self.ippToPspsDb,
                                              self.scratchDb,
                                              id,
                                              batchID)
                    
                batch.run()
   
            # if batch fails, ignore and move on to the next one
            except Exception, e: 
                self.logger.error("EXCEPTION: " + str(e))
                self.logger.errorPair("Problem with this %s batch (%d)" % (batchType, id), "skipping")
                sys.exit(1)
                pass

            if not self.checkClientStatus(): return False
            self.logger.infoPair("checkclient status", "ok")

            self.ippToPspsDb.lockBatchTable()
            self.logger.infoPair("lockbatchtable", "ok")
            self.logger.infoSeparator()
    
        self.ippToPspsDb.unlockTables()
        self.logger.infoPair("Unattempted batches", "%d" % unattemptedCount)

        return True
    
    '''
    Overrides base-class version
    '''
    def printUsage(self):
        super(Loader, self).printUsage("<[init]>")

    
'''
Start of program.
'''
try:
    loader = Loader(sys.argv)
    loader.run()
    loader.exitProgram("completed")
except Exception, e:
    print str(e)
    traceback.print_exc()

