- Timestamp:
- Jan 31, 2013, 4:13:09 PM (13 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopspsdb.py
r35063 r35076 17 17 Constructor 18 18 ''' 19 def __init__(self, logger, config ):19 def __init__(self, logger, config, skychunk): 20 20 if (config.test): 21 21 dbType = "ipptopspsdatabase_test" … … 24 24 super(IppToPspsDb, self).__init__(logger, config, dbType) 25 25 26 self.skychunk = skychunk 27 26 28 self.MAX_FAILS = 5 27 29 … … 33 35 sql = "SELECT DISTINCT batch_id \ 34 36 FROM batch \ 35 WHERE timestamp > '" + self. config.epoch + "' \37 WHERE timestamp > '" + self.skychunk.epoch + "' \ 36 38 AND batch_type = '" + batchType + "' \ 37 AND dvo_db = '" + self. config.dvoLabel + "' \39 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 38 40 AND merged = 1 \ 39 41 AND " + column + " = 0" … … 58 60 sql = "SELECT DISTINCT batch_id \ 59 61 FROM batch \ 60 WHERE timestamp > '" + self. config.epoch + "' \62 WHERE timestamp > '" + self.skychunk.epoch + "' \ 61 63 AND batch_type = '" + batchType + "' \ 62 AND dvo_db = '" + self. config.dvoLabel + "' \64 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 63 65 AND purged = 1 \ 64 66 AND " + column + " = 0" … … 84 86 sql = "SELECT DISTINCT batch_id \ 85 87 FROM batch \ 86 WHERE timestamp > '" + self. config.epoch + "' \88 WHERE timestamp > '" + self.skychunk.epoch + "' \ 87 89 AND batch_type = '" + batchType + "' \ 88 AND dvo_db = '" + self. config.dvoLabel + "' \90 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 89 91 AND (loaded_to_ODM = -1 OR merge_worthy = 1) \ 90 92 AND " + column + " = 0" … … 152 154 sql = "SELECT DISTINCT batch_id \ 153 155 FROM batch \ 154 WHERE timestamp > '" + self. config.epoch + "' \156 WHERE timestamp > '" + self.skychunk.epoch + "' \ 155 157 AND batch_type = '" + batchType + "' \ 156 AND dvo_db = '" + self. config.dvoLabel + "' \158 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 157 159 AND processed = 1\ 158 160 AND loaded_to_datastore != 1 \ … … 181 183 sql = "SELECT batch_id \ 182 184 FROM batch \ 183 WHERE timestamp > '" + self. config.epoch + "' \184 AND dvo_db = '" + self. config.dvoLabel + "' \185 WHERE timestamp > '" + self.skychunk.epoch + "' \ 186 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 185 187 AND loaded_to_datastore = 1 \ 186 188 AND batch_type = '" + batchType + "' \ … … 210 212 sql = "SELECT batch_id \ 211 213 FROM batch \ 212 WHERE timestamp > '" + self. config.epoch + "' \213 AND dvo_db = '" + self. config.dvoLabel + "' \214 WHERE timestamp > '" + self.skychunk.epoch + "' \ 215 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 214 216 AND loaded_to_datastore = 1 \ 215 217 AND batch_type = '" + batchType + "' \ … … 239 241 sql = "SELECT batch_id \ 240 242 FROM batch \ 241 WHERE timestamp > '" + self. config.epoch + "' \242 AND dvo_db = '" + self. config.dvoLabel + "' \243 WHERE timestamp > '" + self.skychunk.epoch + "' \ 244 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 243 245 AND merge_worthy = 1 \ 244 246 AND batch_type = '" + batchType + "' \ … … 266 268 sql = "SELECT DISTINCT batch_id \ 267 269 FROM batch \ 268 WHERE timestamp > '" + self. config.epoch + "' \269 AND dvo_db = '" + self. config.dvoLabel + "' \270 WHERE timestamp > '" + self.skychunk.epoch + "' \ 271 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 270 272 AND " + column + " = " + str(value) 271 273 … … 292 294 FROM batch \ 293 295 WHERE batch_type = '" + batchType + "' \ 294 AND timestamp > '" + self. config.epoch + "' \295 AND dvo_db = '" + self. config.dvoLabel + "' \296 AND timestamp > '" + self.skychunk.epoch + "' \ 297 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 296 298 AND processed = 1 \ 297 299 AND loaded_to_datastore = 1 \ … … 319 321 FROM batch \ 320 322 WHERE batch_type = '" + batchType + "' \ 321 AND timestamp > '" + self. config.epoch + "' \322 AND dvo_db = '" + self. config.dvoLabel + "' \323 AND timestamp > '" + self.skychunk.epoch + "' \ 324 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 323 325 AND " + column + " = " + str(value) 324 326 … … 391 393 WHERE batch_type = '" + batchType + "' \ 392 394 AND processed = 1 \ 393 AND timestamp > '" + self. config.epoch + "' \394 AND dvo_db = '" + self. config.dvoLabel + "' \395 AND timestamp > '" + self.skychunk.epoch + "' \ 396 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 395 397 AND timestamp BETWEEN now() - INTERVAL " + interval + " AND now()" 396 398 … … 416 418 WHERE batch_type = '" + batchType + "' \ 417 419 AND loaded_to_datastore = 1 \ 418 AND timestamp > '" + self. config.epoch + "' \419 AND dvo_db = '" + self. config.dvoLabel + "' \420 AND timestamp > '" + self.skychunk.epoch + "' \ 421 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 420 422 ORDER BY timestamp DESC LIMIT 1" 421 423 … … 515 517 WHERE stage_id = " + str(stage_id) + " \ 516 518 AND batch_type = '" + batchType + "' \ 517 AND dvo_db = '" + self. config.dvoLabel + "' \519 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 518 520 AND batch_type != 'IN' \ 519 521 AND timestamp BETWEEN now() - INTERVAL 12 HOUR AND now()" … … 558 560 WHERE stage_id = " + str(stage_id) + " \ 559 561 AND batch_type = '" + batchType + "' \ 560 AND timestamp > '" + self. config.epoch + "' \561 AND dvo_db = '" + self. config.dvoLabel + "' \562 AND timestamp > '" + self.skychunk.epoch + "' \ 563 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 562 564 AND batch_type != 'IN' \ 563 565 AND processed = 1" … … 583 585 WHERE stage_id = " + str(stage_id) + " \ 584 586 AND batch_type = '" + batchType + "' \ 585 AND timestamp > '" + self. config.epoch + "' \586 AND dvo_db = '" + self. config.dvoLabel + "' \587 AND timestamp > '" + self.skychunk.epoch + "' \ 588 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 587 589 AND processed = -1" 588 590 … … 606 608 FROM batch \ 607 609 WHERE batch_type = '" + batchType + "' \ 608 AND timestamp > '" + self. config.epoch + "' \609 AND dvo_db = '" + self. config.dvoLabel + "' \610 AND timestamp > '" + self.skychunk.epoch + "' \ 611 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 610 612 AND processed = -1 \ 611 613 GROUP BY stage_id HAVING numoccur >= " + str(self.MAX_FAILS) … … 638 640 batchID = -1; 639 641 640 if self. config.force or \642 if self.skychunk.force or \ 641 643 (not self.alreadyProcessed(batchType, stageID) \ 642 644 and not self.processingNow(batchType, stageID) \ … … 654 656 '" + batchType + "', \ 655 657 " + str(stageID) + ", \ 656 '" + self. config.survey + "', \657 '" + self. config.dvoLabel + "', \658 '" + self. config.datastoreProduct + "' \658 '" + self.skychunk.survey + "', \ 659 '" + self.skychunk.dvoLabel + "', \ 660 '" + self.skychunk.datastoreProduct + "' \ 659 661 )" 660 662 self.logger.infoPair("heather:","sql") … … 753 755 754 756 ''' 755 Sets the configfield for this client756 ''' 757 def set ConfigForThisClient(self, config, host, pid):758 759 self.execute("UPDATE clients SET config = '" + config+ "' \757 Sets the skychunk field for this client 758 ''' 759 def setSkychunkForThisClient(self, skychunk, host, pid): 760 761 self.execute("UPDATE clients SET skychunk = '" + skychunk + "' \ 760 762 WHERE host = '" + host + "' \ 761 763 AND pid = " + str(pid)) 762 764 763 765 ''' 764 Sets the configfield for a set of loader clients765 ''' 766 def set ConfigForLoaders(self, config, ids):766 Sets the skychunk field for a set of loader clients 767 ''' 768 def setSkychunkForLoaders(self, skychunk, ids): 767 769 768 770 for id in ids: 769 self.execute("UPDATE clients SET config = '" + config+ "' \771 self.execute("UPDATE clients SET skychunk = '" + skychunk + "' \ 770 772 WHERE type = 'loader.py' \ 771 773 AND id = " + str(id)) … … 801 803 ''' 802 804 def insertClient(self, type, host, pid): 803 self.execute("INSERT INTO clients (timestamp, type, host, pid, config) VALUES (now(), '" + type + "', '" + host + "', " + str(pid) + ", '" + self.config.name + "')")805 self.execute("INSERT INTO clients (timestamp, type, host, pid, skychunk) VALUES (now(), '" + type + "', '" + host + "', " + str(pid) + ", '" + self.skychunk.name + "')") 804 806 805 807 ''' … … 845 847 846 848 ''' 847 Returns a list of available configs848 ''' 849 def getActive ConfigList(self):849 Returns a list of available skychunks 850 ''' 851 def getActiveSkychunkList(self): 850 852 851 sql = "SELECT DISTINCT name FROM configWHERE active = 1 ORDER BY name"852 853 configs = []854 try: 855 rs = self.executeQuery(sql) 856 while (rs.next()): configs.append(rs.getString(1))857 except: 858 self.logger.errorPair("Can't get confignames", sql)853 sql = "SELECT DISTINCT name FROM skychunk WHERE active = 1 ORDER BY name" 854 855 skychunks = [] 856 try: 857 rs = self.executeQuery(sql) 858 while (rs.next()): skychunks.append(rs.getString(1)) 859 except: 860 self.logger.errorPair("Can't get skychunk names", sql) 859 861 finally: 860 862 rs.close() 861 863 862 return configs863 864 ''' 865 Prompts user for info about a new or existing configand lets them neter the various details866 ''' 867 def edit Config(self):864 return skychunks 865 866 ''' 867 Prompts user for info about a new or existing skychunk and lets them neter the various details 868 ''' 869 def editSkychunk(self): 868 870 869 871 print " ********************************************************************" 870 print " * Configeditor"872 print " * Skychunk editor" 871 873 print " *" 872 response = raw_input(" * Name for new config, or existing configto edit? ")874 response = raw_input(" * Name for new skychunk, or existing skychunk to edit? ") 873 875 if response == "": return 874 self. config.name = response875 876 # attempt to insert new config, if it already exists then carry on with update877 sql = "INSERT INTO config (name) VALUES ('" + self.config.name + "')"876 self.skychunk.name = response 877 878 # attempt to insert new skychunk, if it already exists then carry on with update 879 sql = "INSERT INTO skychunk (name) VALUES ('" + self.skychunk.name + "')" 878 880 try: 879 881 self.execute(sql) 880 882 except: pass 881 883 882 # for each column in the configtable883 sql = "SHOW COLUMNS FROM config"884 # for each column in the skychunk table 885 sql = "SHOW COLUMNS FROM skychunk" 884 886 rs = self.executeQuery(sql) 885 887 while (rs.next()): … … 895 897 # get current value for this field, if any 896 898 try: 897 rs2 = self.executeQuery("SELECT " + field + " FROM config WHERE name = '" + self.config.name + "'")899 rs2 = self.executeQuery("SELECT " + field + " FROM skychunk WHERE name = '" + self.skychunk.name + "'") 898 900 rs2.first() 899 901 default = rs2.getString(1) … … 912 914 if type.find("varchar") != -1 or type.find("timestamp") != -1: quotes = "'" 913 915 else: quotes = "" 914 self.execute("UPDATE config SET " + field + " = " + quotes + response + quotes + " WHERE name = '" + self.config.name + "'")916 self.execute("UPDATE skychunk SET " + field + " = " + quotes + response + quotes + " WHERE name = '" + self.skychunk.name + "'") 915 917 916 918 print " *" … … 918 920 919 921 ''' 920 Returns configfor this client921 ''' 922 def get ConfigForThisClient(self, host, pid):923 924 sql = "SELECT config\922 Returns skychunk for this client 923 ''' 924 def getSkychunkForThisClient(self, host, pid): 925 926 sql = "SELECT skychunk \ 925 927 FROM clients \ 926 928 WHERE host = '" + host + "' \ … … 930 932 rs = self.executeQuery(sql) 931 933 rs.first() 932 self. config.name = rs.getString(1)933 except: 934 self.logger.errorPair("No configset for", "%s (pid=%d)" % (host, pid))935 self. config.isLoaded = False934 self.skychunk.name = rs.getString(1) 935 except: 936 self.logger.errorPair("No skychunk set for", "%s (pid=%d)" % (host, pid)) 937 self.skychunk.isLoaded = False 936 938 return False 937 939 … … 939 941 940 942 ''' 941 Reads config from the database and populates Configobject942 ''' 943 def read Config(self, host, pid):944 945 # first get configdefined for this client946 if not self.get ConfigForThisClient(host, pid): return False947 948 # now load that config943 Reads skychunk from the database and populates Skychunk object 944 ''' 945 def readSkychunk(self, host, pid): 946 947 # first get skychunk defined for this client 948 if not self.getSkychunkForThisClient(host, pid): return False 949 950 # now load that skychunk 949 951 sql = "SELECT \ 950 952 datastore_product \ … … 970 972 ,queue_OB \ 971 973 ,parallel \ 972 FROM config\973 WHERE name = '" + self. config.name + "'"974 975 self. config.batchTypes = []974 FROM skychunk \ 975 WHERE name = '" + self.skychunk.name + "'" 976 977 self.skychunk.batchTypes = [] 976 978 try: 977 979 rs = self.executeQuery(sql) 978 980 979 981 rs.first() 980 self. config.datastoreProduct = rs.getString(1)981 self. config.datastoreType = rs.getString(2)982 if rs.getInt(3) == 1: self. config.datastorePublishing = True983 else: self. config.datastorePublishing = False984 self. config.dvoLabel = rs.getString(4)982 self.skychunk.datastoreProduct = rs.getString(1) 983 self.skychunk.datastoreType = rs.getString(2) 984 if rs.getInt(3) == 1: self.skychunk.datastorePublishing = True 985 else: self.skychunk.datastorePublishing = False 986 self.skychunk.dvoLabel = rs.getString(4) 985 987 986 988 # if dvoLabel is null is can break queries, so set to something 987 if not self. config.dvoLabel: self.config.dvoLabel = "none"988 self. config.dvoLocation = rs.getString(5)989 self. config.minRa = rs.getDouble(6)990 self. config.maxRa = rs.getDouble(7)991 self. config.minDec = rs.getDouble(8)992 self. config.maxDec = rs.getDouble(9)993 994 self. config.boxSize = rs.getDouble(10)995 self. config.halfBox = self.config.boxSize/2.0996 self. config.boxSizeWithBorder = self.config.boxSize + (self.config.BORDER * 2)997 998 self. config.basePath = rs.getString(11)999 self. config.dataRelease = rs.getInt(12)1000 1001 self. config.deleteLocal = rs.getInt(13)1002 self. config.deleteDatastore = rs.getInt(14)1003 self. config.deleteDxLayer = rs.getInt(15)1004 1005 self. config.epoch = rs.getString(16)1006 1007 self. config.survey = rs.getString(17)1008 self. config.pspsSurvey = rs.getString(18)1009 1010 if rs.getInt(19) == 1: self. config.batchTypes.append("P2")1011 if rs.getInt(20) == 1: self. config.batchTypes.append("ST")1012 if rs.getInt(21) == 1: self. config.batchTypes.append("OB")1013 1014 self. config.force = True # TODO1015 self. config.parallel = False # TODO1016 1017 if rs.getInt(22) == 1: self. config.parallel = True1018 1019 if self. config.parallel: print "USING parallel"1020 self. config.isLoaded = True1021 except: 1022 self.logger.errorPair("Could not read config with name", self.config.name)1023 self. config.isLoaded = False1024 1025 return self. config.isLoaded989 if not self.skychunk.dvoLabel: self.skychunk.dvoLabel = "none" 990 self.skychunk.dvoLocation = rs.getString(5) 991 self.skychunk.minRa = rs.getDouble(6) 992 self.skychunk.maxRa = rs.getDouble(7) 993 self.skychunk.minDec = rs.getDouble(8) 994 self.skychunk.maxDec = rs.getDouble(9) 995 996 self.skychunk.boxSize = rs.getDouble(10) 997 self.skychunk.halfBox = self.skychunk.boxSize/2.0 998 self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2) 999 1000 self.skychunk.basePath = rs.getString(11) 1001 self.skychunk.dataRelease = rs.getInt(12) 1002 1003 self.skychunk.deleteLocal = rs.getInt(13) 1004 self.skychunk.deleteDatastore = rs.getInt(14) 1005 self.skychunk.deleteDxLayer = rs.getInt(15) 1006 1007 self.skychunk.epoch = rs.getString(16) 1008 1009 self.skychunk.survey = rs.getString(17) 1010 self.skychunk.pspsSurvey = rs.getString(18) 1011 1012 if rs.getInt(19) == 1: self.skychunk.batchTypes.append("P2") 1013 if rs.getInt(20) == 1: self.skychunk.batchTypes.append("ST") 1014 if rs.getInt(21) == 1: self.skychunk.batchTypes.append("OB") 1015 1016 self.skychunk.force = True # TODO 1017 self.skychunk.parallel = False # TODO 1018 1019 if rs.getInt(22) == 1: self.skychunk.parallel = True 1020 1021 if self.skychunk.parallel: print "USING parallel" 1022 self.skychunk.isLoaded = True 1023 except: 1024 self.logger.errorPair("Could not read skychunk with name", self.skychunk.name) 1025 self.skychunk.isLoaded = False 1026 1027 return self.skychunk.isLoaded 1026 1028 1027 1029 ''' … … 1031 1033 1032 1034 sql = "SELECT id FROM box \ 1033 WHERE config = '" + self.config.name + "' \1035 WHERE skychunk = '" + self.skychunk.name + "' \ 1034 1036 AND ra_center = " + str(ra) + " \ 1035 1037 AND dec_center = " + str(dec) + " \ 1036 AND box_side = " + str(self. config.boxSize)1038 AND box_side = " + str(self.skychunk.boxSize) 1037 1039 1038 1040 id = -1 … … 1050 1052 ''' 1051 1053 def removeAllBoxes(self): 1052 self.execute("DELETE FROM box WHERE config = '" + self.config.name + "'")1054 self.execute("DELETE FROM box WHERE skychunk = '" + self.skychunk.name + "'") 1053 1055 1054 1056 ''' … … 1063 1065 1064 1066 sql = "INSERT INTO box \ 1065 ( config, ra_center, dec_center, box_side) \1067 (skychunk, ra_center, dec_center, box_side) \ 1066 1068 VALUES \ 1067 ('" + self. config.name + "', " + str(ra) + ", " + str(dec) + ", " + str(self.config.boxSize) + ")"1069 ('" + self.skychunk.name + "', " + str(ra) + ", " + str(dec) + ", " + str(self.skychunk.boxSize) + ")" 1068 1070 1069 1071 self.execute(sql) … … 1104 1106 FROM box \ 1105 1107 JOIN pending ON (id = box_id) \ 1106 WHERE config = '" + self.config.name + "' \1108 WHERE skychunk = '" + self.skychunk.name + "' \ 1107 1109 AND batch_type = '" + batchType + "' \ 1108 1110 AND ra_center NOT IN \ 1109 1111 (SELECT ra_center \ 1110 1112 FROM stripe \ 1111 WHERE config = '" + self.config.name + "') \1113 WHERE skychunk = '" + self.skychunk.name + "') \ 1112 1114 GROUP BY ra_center LIMIT 1" 1113 1115 … … 1124 1126 FROM box \ 1125 1127 JOIN pending ON (id = box_id) \ 1126 WHERE config = '" + self.config.name + "' \1128 WHERE skychunk = '" + self.skychunk.name + "' \ 1127 1129 AND batch_type = '" + batchType + "' \ 1128 1130 GROUP BY ra_center \ … … 1144 1146 WHERE ra_center = " + str(raCenter) + " \ 1145 1147 AND batch_type = '" + batchType + "' \ 1146 AND config = '" + self.config.name + "' \1148 AND skychunk = '" + self.skychunk.name + "' \ 1147 1149 ORDER BY dec_center DESC" 1148 1150 … … 1177 1179 # now insert new stripe entry 1178 1180 sql = "INSERT INTO stripe \ 1179 (client_id, config, ra_center) \1180 SELECT clients.id, clients. config, " + str(raCenter) + " \1181 (client_id, skychunk, ra_center) \ 1182 SELECT clients.id, clients.skychunk, " + str(raCenter) + " \ 1181 1183 FROM clients \ 1182 1184 WHERE host = '" + host + "' \ … … 1206 1208 1207 1209 ''' 1208 Returns ids for pending items for this config1210 Returns ids for pending items for this skychunk 1209 1211 ''' 1210 1212 def getPendingIds(self, batchType): … … 1213 1215 FROM pending \ 1214 1216 JOIN box ON (box_id = id) \ 1215 WHERE config = '" + self.config.name + "' \1217 WHERE skychunk = '" + self.skychunk.name + "' \ 1216 1218 AND batch_type = '" + batchType + "'" 1217 1219 … … 1222 1224 rs.close() 1223 1225 except: 1224 self.logger.errorPair("Can't get pending ids for this config", sql)1226 self.logger.errorPair("Can't get pending ids for this skychunk", sql) 1225 1227 1226 1228 return ids … … 1253 1255 WHERE stage_id = " + str(stageID) + " \ 1254 1256 AND batch_type = '" + batchType + "' \ 1255 AND config = '" + self.config.name + "'"1257 AND skychunk = '" + self.skychunk.name + "'" 1256 1258 1257 1259 self.execute(sql) 1258 1260 1259 1261 ''' 1260 Writes density-plot data to file for pending stuff over sky for given config1262 Writes density-plot data to file for pending stuff over sky for given skychunk 1261 1263 ''' 1262 1264 def createPendingDensityPlotData(self, batchType, DATFILE): … … 1264 1266 sql = "SELECT id, ra_center, dec_center \ 1265 1267 FROM box \ 1266 WHERE config = '" + self.config.name + "' \1268 WHERE skychunk = '" + self.skychunk.name + "' \ 1267 1269 ORDER BY ra_center, dec_center" 1268 1270 … … 1317 1319 def getItemsInThisThisBox(self, ra, dec): 1318 1320 1319 halfSide = self. config.boxSize/2.01321 halfSide = self.skychunk.boxSize/2.0 1320 1322 minRa = ra-halfSide 1321 1323 maxRa = ra+halfSide … … 1346 1348 FROM stripe \ 1347 1349 JOIN clients ON (client_id = id) \ 1348 WHERE clients. config = '" + self.config.name + "' \1350 WHERE clients.skychunk = '" + self.skychunk.name + "' \ 1349 1351 ORDER BY host, started" 1350 1352
Note:
See TracChangeset
for help on using the changeset viewer.
