Package install :: Package MoSTBioDat :: Package DataBase :: Package Connect :: Module MoSTBioDatDBSupport
[hide private]
[frames] | no frames]

Source Code for Module install.MoSTBioDat.DataBase.Connect.MoSTBioDatDBSupport

  1  #!/usr/bin/env python 
  2  ################################################### 
  3  # MoSTBioDatDBSupport.py:                         # 
  4  # Supportive classes and methods for MoSTBioDatDB # 
  5  ################################################### 
  6   
  7  ###################################################### 
  8  # Copyright (c) 2007-2008 Andrzej Bak                # 
  9  # ARC Seibersdorf & University of Silesia            # 
 10  # Author: Andrzej Bak <Andrzej.Bak@us.edu.pl>        # 
 11  # License: GNU General Public License, version: 3    # 
 12  # URL: http://chemoinformatyka.us.edu.pl/mostbiodat/ # 
 13  # Version: 1, 06.01.2010                             # 
 14  ###################################################### 
 15   
 16  try: 
 17      import sys 
 18      import numpy 
 19      import shelve 
 20      import os 
 21      import stat 
 22      import anydbm 
 23      import cPickle 
 24      from MoSTBioDat.Log import MoSTBioDatLog 
 25      from time import strftime,localtime 
 26      import tarfile 
 27      import glob 
 28  except ImportError,e: 
 29      print 'Error: %s' %e 
 30      sys.exit(1) 
 31  ######## dictionary to array #################### 
32 -def dict2array(rowdict,row):
33 """ 34 from cursor dictionary create array 35 INPUT: 36 row cursor dictionary 37 row counter 38 OUTPUT: 39 array of objects 40 """ 41 rowarray=numpy.array([],dtype=object)#numpy object array 42 if row==0: 43 colnames=numpy.array([],dtype=object)#addition head row 44 colnames=numpy.append(colnames,['number/name'])#first descriptive row item 45 counter=1 46 for key in rowdict.keys(): 47 colnames=numpy.append(colnames,key)#add column name to descriptive row 48 counter+=1 49 colnames.resize(1,counter) 50 rowarray=numpy.append(rowarray,row+1) 51 counter=1 52 for value in rowdict.values():#get values 53 rowarray=numpy.append(rowarray,value) 54 counter+=1 55 rowarray.resize(1,counter) 56 if row==0: 57 rowarray=numpy.concatenate((colnames,rowarray)) 58 return rowarray
59
60 -def descolumn(columndesclist):
61 columndesc={} 62 """ 63 MySQL FIELD_TYPE Constants 64 These constants represent the various column (field) types that are 65 supported by MySQL. 66 INPUT: 67 column description list 68 OUTPUT: 69 column description 70 """ 71 Num2StrFieldConversions={ 72 0:'DECIMAL', 73 1:'TINY', 74 2:'SHORT', 75 3:'LONG', 76 4:'FLOAT', 77 5:'DOUBLE', 78 6:'NULL', 79 7:'TIMESTAMP', 80 8:'LONGLONG', 81 9:'INT24', 82 10:'DATE', 83 11:'TIME', 84 12:'DATETIME', 85 13:'YEAR', 86 14:'NEWDATE', 87 247:'ENUM' , 88 248:'SET', 89 249:'TINY_BLOB', 90 250:'MEDIUM_BLOB', 91 251:'LONG_BLOB', 92 252:'BLOB', 93 253:'VAR_STRING', 94 254:'STRING', 95 } 96 for item in columndesclist: 97 try: 98 num2str=Num2StrFieldConversions[item[1]]#change numeric code to string 99 except KeyError: 100 num2str='UNKNOWN' 101 columndesc[item[0]]=num2str 102 return columndesc
103
104 -def filterKeys(dictionary,keys,key):
105 """ 106 check input settings 107 """ 108 if key not in keys: 109 del dictionary[key] 110 return False 111 else: 112 return True
113
114 -class PickleArray(object):
115 """ 116 cPickle array - user cache memory 117 INPUT: 118 query - str - SQL query 119 cachepath - str - path to cache directory, default - current working directory 120 cachefilename - str - cache file names, default query 121 cachedictfilename - str - dictionary cache file name, default - dictQuery 122 numcachefiles - int - number cache files 123 temporary - str - temporary cache directory, default /tmp/Query 124 """
125 - def __init__(self,query,**kwargs):
126 self.query=query 127 self.kwargs={} 128 self.kwargs=kwargs 129 130 if not self.kwargs.has_key('cachepath'): 131 self.kwargs.setdefault('cachepath','/tmp/Query') 132 if not self.kwargs.has_key('cachefilename'): 133 self.kwargs.setdefault('cachefilename','query') 134 if not self.kwargs.has_key('cachedictfilename'): 135 self.kwargs.setdefault('cachedictfilename','dictQuery') 136 if not self.kwargs.has_key('protocol'): 137 self.kwargs.setdefault('protocol',-1) 138 if not self.kwargs.has_key('numcachefiles'): 139 self.kwargs.setdefault('numcachefiles',100000) 140 if not self.kwargs.has_key('temporary'): 141 self.kwargs.setdefault('temporary','/tmp/Query') 142 self.createdict()
143
144 - def createdict(self):
145 """ 146 create cache dictionary 147 INPUT: 148 class object 149 OUTPUT: 150 """ 151 if self.kwargs['cachepath']=='': 152 self.kwargs['cachepath']=self.getlocalDir() 153 if not os.path.isdir(self.kwargs['cachepath']): 154 try: 155 os.makedirs(self.kwargs['cachepath']) 156 except Exception,error: 157 print 'Error: %s, %s for %s' %(error[0],error[1],self.kwargs['cachepath']) 158 print 'Cache files in %s' %self.kwargs['temporary']#create directory in temporary path 159 self.kwargs['cachepath']=self.kwargs['temporary'] 160 if not os.path.exists(self.kwargs['cachepath']): 161 os.makedirs(self.kwargs['cachepath']) 162 else: 163 if not os.access(self.kwargs['cachepath'],os.W_OK): 164 print 'Error: Permission denied! Unable to write in %s' %self.kwargs['cachepath'] 165 self.kwargs['cachepath']=self.kwargs['temporary'] 166 if not os.path.exists(self.kwargs['cachepath']): 167 os.makedirs(self.kwargs['cachepath']) 168 169 elif not os.access(self.kwargs['cachepath'],os.W_OK): 170 print 'Error: Permission denied! Unable to write in %s' %self.kwargs['cachepath'] 171 sys.exit(1) 172 173 self.dictfullfilename=os.path.join(self.kwargs['cachepath'],self.kwargs['cachedictfilename']) 174 ### dictionary full filename path 175 print 'Cache dictionary: %s' %self.dictfullfilename
176
177 - def query2dict(self,log):
178 """ 179 insert query to cache dictionary 180 flag - boolean - check if search query in dictionary, default True 181 INPUT: 182 class object 183 OUTPUT: 184 """ 185 try: 186 dbquery=shelve.open(self.dictfullfilename,protocol=self.kwargs['protocol']) 187 except anydbm.error,error: 188 print 'Error: %s, %s' %(error,self.dictfullfilename) 189 log.exception('Error: %s %s',error,self.dictfullfilename) 190 else: 191 numfile=int(self.backupLocalFile().split('.')[1])#get highest file_NUMBER 192 if numfile==self.kwargs['numcachefiles']:#compare it with user limit 193 print 'Cache overflowed! Creating tar compressed archive' 194 log.info('Cache overflowed!') 195 self.makeTar(log)#create tar repository 196 dbquery.close() 197 try: 198 dbquery=shelve.open(self.dictfullfilename,protocol=self.kwargs['protocol']) 199 ### open new shelve dictionary 200 except anydbm.error,error: 201 print 'Error: %s, %s' %(error,self.dictfullfilename) 202 log.exception('Error: %s %s',error,self.dictfullfilename) 203 newfilename=self.backupLocalFile()#create new cPickle filename 204 dbquery[self.query]=newfilename#insert to shelve dictionary 205 dbquery.close()
206
207 - def getlocalDir(self):
208 """ 209 get local working directory 210 """ 211 return os.path.abspath(os.curdir)
212
213 - def backupLocalFile(self):
214 """ 215 create cache file, f.e. query.00001 216 INPUT: 217 class object 218 OUTPUT: 219 file full name 220 """ 221 numfile=1 222 def numbackupLocalFile(version): 223 return os.path.join(self.kwargs['cachepath'],'%s.%05d'%(self.kwargs['cachefilename'],numfile))
224 filefullname=numbackupLocalFile(numfile) 225 while os.path.exists(filefullname): 226 numfile+=1 227 filefullname=numbackupLocalFile(numfile) 228 return filefullname#next number in directory
229
230 - def getQueryFilePath(self):
231 """ 232 get query file path 233 INPUT: 234 class object 235 OUTPUT: 236 query cache file path 237 """ 238 try: 239 dbquery=shelve.open(self.dictfullfilename,'r',protocol=self.kwargs['protocol']) 240 except anydbm.error,error: 241 print 'Error: %s for %s' %(error,self.kwargs['cachepath']) 242 else: 243 queryfilepath=dbquery.get(self.query,'')#get cPickle cache file path 244 dbquery.close() 245 return queryfilepath
246
247 - def array2cpickle(self,obj,log):
248 """ 249 create cPickle from array 250 INPUT: 251 class object 252 obj - object 253 OUTPUT: 254 """ 255 queryfilepath=self.getQueryFilePath() #get path to cache file from cache dict 256 ### necessary because new query already in shelve dictionary 257 if queryfilepath=='': 258 print 'Can not create cPickle in %s' %self.kwargs['cachepath'] 259 log.info('Can not create cPickle in %s',self.kwargs['cachepath']) 260 print 'Cache files in %s directory' %self.kwargs['temporary'] 261 log.info('Cache files in %s directory',self.kwargs['temporary']) 262 queryfilepath=os.path.join(self.kwargs['temporary'],self.kwargs['cachefilename'])# 263 ### temporary directory and cache file path 264 try: 265 file=open(queryfilepath,'wb') 266 cPickle.dump(obj,file) 267 file.close() 268 print 'Query in cache file: %s' %queryfilepath 269 log.info('Query in %s',queryfilepath) 270 except cPickle.PickleError, error: 271 print 'Error: %s' %error
272
273 - def cpickle2array(self):
274 """ 275 create array from cPickle 276 INPUT: 277 class object 278 OUTPUT: 279 obj - object 280 """ 281 if not os.path.exists(self.dictfullfilename): 282 return None,None 283 else: 284 try: 285 dbquery=shelve.open(self.dictfullfilename,'r',protocol=self.kwargs['protocol']) 286 except anydbm.error,error: 287 print 'Error: %s for %s' %(error,self.dictfullfilename) 288 else: 289 queryfilepath=dbquery.get(self.query,None) 290 dbquery.close() 291 if queryfilepath:#query filename path exists in dictionary 292 if os.path.exists(queryfilepath):#exist in filesystem 293 try: 294 file=open(queryfilepath,'rb') 295 obj=cPickle.load(file) 296 file.close() 297 return obj,queryfilepath 298 except cPickle.PickleError, error: 299 print 'Error: %s' %error 300 return None,None 301 else: 302 return None,None 303 else: 304 return None,None
305
306 - def rmfile(self,path):
307 """ 308 remove file 309 INPUT: 310 path - str- file path 311 """ 312 if os.path.exists(path): 313 try: 314 os.remove(path) 315 print 'Deleting: %s' %path 316 except OSError,error: 317 print 'Error: %s' %error
318
319 - def showCacheDict(self):
320 """ 321 show dictionary 322 INPUT: 323 class object 324 OUTPUT: 325 """ 326 try: 327 dbquery=shelve.open(self.dictfullfilename,'r',protocol=self.kwargs['protocol']) 328 except anydbm.error,error: 329 print 'Error: %s for %s' %(error,self.kwargs['cachepath']) 330 else: 331 for key,value in dbquery.items(): 332 print 'Query:',key, 'Cache file:',value
333
334 - def makeTar(self,log):
335 """ 336 make tar repository 337 INPUT: 338 class object 339 log - logging handler 340 OUTPUT: 341 """ 342 try: 343 files=glob.glob(os.path.join(self.kwargs['cachepath'],self.kwargs['cachefilename']+'.*')) 344 ### get files in directory with filename.* 345 files.append(os.path.join(self.kwargs['cachepath'],self.kwargs['cachedictfilename'])) 346 ### add shelve dictionary 347 except OSError,e: 348 print 'Error: %s' %e 349 log.exception('Error: %s',e) 350 else: 351 if files==[]: 352 print 'No files to tar' 353 log.info('No files to tar') 354 else: 355 try: 356 datestamp='_'+strftime('%d%m%y',localtime()) 357 tarfilename='query'+datestamp+'.tar.bz2'#creata standard repository name query_data.tar.bz2 358 tarfullfilename=os.path.join(self.kwargs['cachepath'],tarfilename) 359 tar=tarfile.open(tarfullfilename,'w:bz2') 360 print 'Adding files to tar file: %s' %tarfullfilename 361 log.info('Adding files to tar file: %s',tarfullfilename) 362 for file in files: 363 tar.add(file)#add file to repository 364 os.remove(file)#remove file from directory 365 print '%s' %file 366 tar.close() 367 except tarfile.TarError,e: 368 print 'Error: %s' %e 369 log.exception('Error: %s',e)
370 371 ################## End of class #########################
372 -def rmfile(path):
373 """ 374 remove file 375 INPUT: 376 path - str- file path 377 """ 378 if os.path.exists(path): 379 try: 380 os.remove(path) 381 print 'Deleting: %s' %path 382 except OSError,error: 383 print 'Error: %s' %error
384 ### function for searching and retrieving query objects ##
385 -def findQueryCacheFile(dictfullfilename,query):
386 """ 387 find cache file with given query 388 INPUT: 389 dictionary full pathname - str 390 query - SQL query - str 391 OUTPUT: 392 filename path to cache file 393 """ 394 if os.path.exists(dictfullfilename): 395 try: 396 dbquery=shelve.open(dictfullfilename) 397 except anydbm.error,error: 398 print 'Error: %s %s' %(error,dictfullfilename) 399 else: 400 return dbquery.get(query,None)
401
402 -def getQueryFileObject(cachefilepath):
403 """ 404 get query file object from cPickle 405 INPUT: 406 cache filename path - str 407 OUTPUT 408 dictionary - 'desc' - column description 409 'array' - array of objects 410 """ 411 412 if os.path.exists(cachefilepath): 413 try: 414 file=open(cachefilepath,'rb') 415 obj=cPickle.load(file) 416 file.close() 417 return obj 418 except cPickle.PickleError, error: 419 print 'Error: %s' %error
420 ################# End of functions ##################### 421 ############## MAIN ################################ 422 ############ Example of usage ########################## 423 if __name__=='__main__': 424 pass 425