Package install :: Package MoSTBioDat :: Package Transfer :: Module fileFTPtools
[hide private]
[frames] | no frames]

Source Code for Module install.MoSTBioDat.Transfer.fileFTPtools

  1  #!/usr/bin/env python 
  2  ######################################################### 
  3  # fileFTPtools.py:                                      # 
  4  # Supportive classes and functions for the FTP module   # 
  5  ######################################################### 
  6   
  7  ###################################################### 
  8  # Copyright (c) 2007-2008 Andrzej Bak                # 
  9  # ARC Seibersdorf & University of Silesia            # 
 10  # Author: Andrzej Bak <Andrzej.Bak@us.edu.pl>        # 
 11  # License: GNU General Public License, version: 3    # 
 12  # URL: http://chemoinformatyka.us.edu.pl/mostbiodat/ # 
 13  # Version: 1, 06.01.2010                             # 
 14  ###################################################### 
 15   
 16  try: 
 17      import sys 
 18      import os 
 19      import time 
 20      import stat 
 21      import shutil 
 22  except ImportError,error: 
 23      print 'Missing modules ...',error 
 24      sys.exit(1) 
 25  ############ fileFtpTools class ####################### 
26 -class fileFtpTools(object):
27 """ 28 methods operating on file object 29 INPUT: 30 fileFtpInfo - file object 31 listfile - list, list of files to download, default empty 32 """ 33 fullDownload,partialDownload,notDownload=0,1,2
34 - def __init__(self,fileFtpInfo,listfile=[]):
35 self.fileFtpInfo=fileFtpInfo #class object 36 self.listfile=listfile #list of files to download
37
38 - def files2download(self):
39 """ 40 chose files to download 41 INPUT: 42 class object 43 OUTPUT: 44 list of files to download 45 """ 46 return filter(lambda filename: self.compare(filename),self.listfile)
47
48 - def compare(self,filename):
49 """ 50 compare files names 51 INPUT 52 class object 53 filename - str, filename 54 OUTPUT 55 boolean 56 """ 57 filename=os.path.split(filename)[1] 58 filefullname=os.path.join(self.fileFtpInfo.cwd,filename) 59 return filefullname==self.fileFtpInfo.filefullname
60
61 - def getlocalDir(self):
62 """ 63 get local directory 64 INPUT: 65 class object 66 OUTPUT: 67 path to local directory 68 """ 69 return os.path.abspath(os.curdir)
70
71 - def backupLocalFile(self,path,filename):
72 """ 73 create a new file: filename.001 if filename exists 74 INPUT: 75 class object 76 path - str, path to file 77 local filename - str, filename 78 OUTPUT: 79 filename 80 """ 81 numfile=1 82 def numbackupLocalFile(version): 83 return os.path.join(path,'%s.%03d'%(filename,numfile))
84 filefullname=numbackupLocalFile(numfile) # create filename 85 while os.path.exists(filefullname): 86 numfile+=1 87 filefullname=numbackupLocalFile(numfile) #create numer for file 88 return filefullname
89
90 - def downloadFile(self,path=''):
91 """ 92 check the version of the file 93 INPUT: 94 object of the fileFtpInfo class 95 path - str, default '.' 96 OUTPUT: 97 state of the file to be downloaded: 0,1,2 98 """ 99 if path=='': 100 path=os.path.abspath(os.curdir) 101 filename=os.path.join(path,self.fileFtpInfo.filename) 102 if os.path.isfile(filename): 103 localdate,remotedate=os.path.getmtime(filename),self.fileFtpInfo.date 104 localsize,remotesize=os.path.getsize(filename), self.fileFtpInfo.size 105 ### compare date, size of local file with counterpart on remote machine 106 if round(localdate)==round(remotedate): 107 if localsize==remotesize: 108 return fileFtpTools.notDownload 109 else: 110 return fileFtpTools.partialDownload 111 else: 112 newfilename=self.backupLocalFile(path,self.fileFtpInfo.filename) 113 os.rename(filename,newfilename) 114 return fileFtpTools.fullDownload 115 else: 116 return fileFtpTools.fullDownload
117
118 - def makeLocalFile(self,path='',binary=True,append=None):
119 """ 120 create directory/file in the FTP server location 121 INPUT: 122 object of the fileFTpInfo class 123 path - str, path to file, default empty 124 binary - boolean, default - True 125 append - default None 126 OUTPUT: 127 file on the local machine 128 """ 129 if path=='': 130 path=self.getlocalDir() 131 filename=os.path.join(path,self.fileFtpInfo.filename) 132 if not os.path.isdir(path): 133 try: 134 os.makedirs(path) #create directory 135 except IOError,error: 136 print 'Error: %s, %s for %s' %(error[0],error[1],path) 137 sys.exit(1) 138 try: 139 permissions=self.check(binary,'wb','w') #get permission 140 if append and os.path.isfile(filename): 141 permissions+='a'#append permission 142 perm=os.stat(filename)[stat.ST_MODE] 143 if not perm & stat.S_IWUSR: 144 os.chmod(filename,perm|stat.S_IWUSR) 145 return file(filename,permissions) 146 except IOError,error: 147 print 'Error: %s, %s for %s' %(error[0],error[1],path) 148 sys.exit(1)
149
150 - def transferFtpFile(self,connection,path,log,binary=True,callback=None):
151 """ 152 transfer file from remote directory to local machine 153 INPUT: 154 connection - class object 155 path on local machine to download files - str 156 binary - boolean, default True 157 callback - default None 158 OUTPUT: 159 transfer file 160 """ 161 if self.fileFtpInfo.isdir or self.fileFtpInfo.islink: #get only regular file 162 print 'Sorry, %s is not a regular file' %self.fileFtpInfo.filefullname 163 return 164 status=self.downloadFile(path) 165 if status==fileFtpTools.notDownload: 166 print 'The exact copy of the %s exists in %s ' %(self.fileFtpInfo.filename,path) 167 log.info('The exact copy of the %s exists in %s',self.fileFtpInfo.filename,path) 168 return 169 if not callback: 170 filecopy=self.makeLocalFile(path,binary,status==fileFtpTools.partialDownload) 171 ###create local file copy 172 callback=filecopy.write 173 t0=time.time() #set starting time 174 size=connection.size(self.fileFtpInfo.filename) # get file size 175 try: 176 filename=os.path.join(path,self.fileFtpInfo.filename) 177 retrstr='RETR %s' %self.fileFtpInfo.filename 178 if binary: 179 if status==fileFtpTools.partialDownload: 180 print 'Downloading %s ...' %self.fileFtpInfo.filename 181 connection.retrbinary(retrstr,callback,rest=os.path.getsize(filename)) 182 ### retrieve data in binary mode 183 else: 184 print 'Downloading %s ...' %self.fileFtpInfo.filename 185 connection.retrbinary(retrstr,callback) 186 else: 187 connection.retrlines(retrstr,callback) 188 finally: 189 t1=time.time() # set final time 190 dt=t1-t0 191 kbytes=size/1024.0 192 print int(round(kbytes)),'Kbytes received in %.2f sec' %dt 193 if t1>t0: 194 print '(~%d kB/s)' %int(round(kbytes/dt)) 195 if filecopy: 196 filecopy.close() 197 print 'Download done.' 198 log.info('Downloaded %s in %.2f sec',self.fileFtpInfo.filename,dt) 199 os.utime(filename,(self.fileFtpInfo.date,self.fileFtpInfo.date)) # set time 200 os.chmod(filename,self.fileFtpInfo.mode) # set permissions
201
202 - def check(self,first,second,third):
203 """ 204 counterpart of C x?y:z 205 """ 206 if first: 207 return second 208 else: 209 return third
210 211 ####################### End of the class ############################
212 -def rmlocalDir(path):
213 """ 214 remove local directory tree 215 INPUT: 216 path to directory to be removed 217 """ 218 if os.path.exists(path): 219 try: 220 shutil.rmtree(path) # remove whole path tree 221 print 'Deleting: %s' %path 222 except OSError,error: 223 print error 224 else: 225 print 'No such directory: %s' %path
226 227 ############## Supportive function for lsFtp method ################
228 -def fileFilter(file):
229 """ 230 remove '.' and '..' from listing 231 INPUT: 232 file - str, filename 233 OUTPUT: 234 boolean True or False 235 """ 236 if not (file.startswith('-') or file.startswith('d') or file.startswith('l')): 237 return False 238 else: 239 filename=file.split()[-1]#take the filename 240 filename=os.path.split(filename)[1] 241 if filename.startswith('.'): 242 return False 243 else: 244 return True
245
246 -def fileInfo(cwd,line,flag):
247 """ 248 detailed file description's parser 249 INPUT: 250 cwd - current working directory 251 line - line 252 flag - boolean, if True print line 253 ftp file's description 254 OUTPUT: 255 file object 256 """ 257 fullmode, links, owner, group, size, rest = line.split(None,5) 258 isdir, islink=parseFtpPerm(fullmode).str2perm() 259 mode=parseFtpPerm(fullmode).str2fullnum() # change string to numeric mode 260 datestr,filename=rest[:12],rest.split()[-1] 261 filename=os.path.split(filename)[1] 262 date=parseFtpDate(datestr).parseDate() # change date to numeric mode 263 fileobj=fileFtpInfo(cwd,filename,fullmode,isdir,islink,mode,int(links,),owner,group,int(size),datestr,date,line) 264 if flag: 265 print fileobj.line 266 return fileobj
267 268 ################ parse Ftp permission class ##########################
269 -class parseFtpPerm(object):
270 """ 271 parse file's permission attributes 272 INPUT: 273 fullmode - str, file's permission string 274 OUTPUT: 275 class object 276 """
277 - def __init__(self,fullmode):
278 self.str=fullmode 279 self._rNum,self._wNum,self._xNum,self._sNum=4,2,1,0 280 self._rStr,self._wStr,self._xStr,self._sStr='r','w','x','-'
281
282 - def str2perm(self):
283 """ 284 check whether file is link or directory 285 """ 286 return self.str[0]=='d',self.str[0]=='l'
287
288 - def str2fullnum(self):
289 u,g,o=self.str[1:4],self.str[4:7],self.str[7:10] 290 return self.str2num(u)<<6|self.str2num(g)<<3|self.str2num(o)
291
292 - def str2num(self,userperm):
293 """ 294 sum the string to the numeric 295 """ 296 r,w,x=userperm[0]==self._rStr,userperm[1]==self._wStr,userperm[2]==self._xStr 297 return self.check(r,self._rNum,self._sNum)|self.check(w,self._wNum,self._sNum)|self.check(x,self._xNum,self._sNum)
298
299 - def rwx(self,n, suid, sticky=False):
300 if suid: 301 suid = 2 302 out = '-r'[n >> 2] + '-w'[(n >> 1) & 1] 303 if sticky: 304 out += '-xTt'[suid + (n & 1)] 305 else: 306 out += '-xSs'[suid + (n & 1)] 307 return out
308
309 - def num2fullstr(self,mode):
310 """ 311 change numeric mode to full string 312 """ 313 if mode is not None: 314 kind = stat.S_IFMT(mode) 315 if kind == stat.S_IFIFO: 316 ks = 'p' 317 elif kind == stat.S_IFCHR: 318 ks = 'c' 319 elif kind == stat.S_IFDIR: 320 ks = 'd' 321 elif kind == stat.S_IFBLK: 322 ks = 'b' 323 elif kind == stat.S_IFREG: 324 ks = '-' 325 elif kind == stat.S_IFLNK: 326 ks = 'l' 327 elif kind == stat.S_IFSOCK: 328 ks = 's' 329 else: 330 ks = '?' 331 ks += self.rwx((mode & 0700) >> 6, mode & stat.S_ISUID) 332 ks += self.rwx((mode & 070) >> 3, mode & stat.S_ISGID) 333 ks += self.rwx(mode & 7, mode & stat.S_ISVTX, True) 334 else: 335 ks = '?---------' 336 return ks
337
338 - def check(self,first,second,third):
339 """ 340 counterpart of the C x?y:z syntax 341 """ 342 if first: 343 return second 344 else: 345 return third
346 347 ################### End of the class ################################# 348 349 ######################## parseFtpDate class ##########################
350 -class parseFtpDate(object):
351 """ 352 parse the date 353 INPUT: 354 date string 355 """
356 - def __init__(self,date):
357 self.currForm='%b %d %H:%M' 358 self.prevForm='%b %d %Y' 359 self.unionForm ='%Y-%m-%d-%H:%M' 360 self.date=date
361
362 - def parseCurrYear(self):
363 """ 364 parse current year 365 """ 366 try: 367 datewith1900 = time.strptime( self.date, self.currForm ) 368 currentYear = time.localtime()[0] 369 except ValueError,error: 370 print 'Error: ',error 371 else: 372 return self.updateTuple(datewith1900,0,currentYear)
373
374 - def parsePrevYear(self):
375 """ 376 parse previous year 377 """ 378 try: 379 olddate=time.strptime( self.date, self.prevForm ) 380 except ValueError: 381 pass 382 else: 383 return olddate
384
385 - def dateParser(self):
386 """ 387 choose the date parser depending on the date format 388 """ 389 if ':' in self.date: 390 return self.parseCurrYear() 391 else: 392 return self.parsePrevYear() 393
394 - def parseDate(self):
395 return time.mktime(self.dateParser()) 396
397 - def displayDate(self,date):
398 """ 399 display date 400 """ 401 date_struct, curr_struct = time.localtime( date ), time.localtime() 402 date_year, curr_year = date_struct[0], curr_struct[0] 403 year_fmt = self.check( date_year == curr_year, self.currForm, self.prevForm ) 404 return time.strftime( year_fmt, date_struct )
405
406 - def updateTuple(self,t,i,x):
407 l = list(t) 408 return tuple( l[:i] + [x] + l[i+1:] )
409
410 - def check(self,first,second,third):
411 """ 412 counterpart of the C x?y:z syntax 413 """ 414 if first: 415 return second 416 else: 417 return third
418 #################### End of the class ########################################## 419 420 ##################### fileFptInfo class ######################################
421 -class fileFtpInfo(object):
422 """ 423 create file object 424 INPUT: 425 file attributes: 426 cwd 427 filename 428 modeStr 429 isdir 430 islink 431 mode 432 links 433 owner 434 group 435 size 436 dateStr 437 date 438 line 439 OUTPUT: 440 file object 441 """
442 - def __init__(self,cwd,filename,modeStr,isdir,islink,mode,links,owner,group,size,dateStr,date,line):
443 self.cwd=cwd 444 self.filename=filename 445 self.modeStr=modeStr 446 self.isdir=isdir 447 self.islink=islink 448 self.mode=mode 449 self.links=links 450 self.owner=owner 451 self.group=group 452 self.size=size 453 self.dateStr=dateStr 454 self.date=date 455 self.filefullname=os.path.join(cwd,filename) 456 self.line=line 457 self.age=time.mktime(time.localtime())
458 ################### End of the class ################ 459 ################### MAIN ############################ 460 if __name__=='__main__': 461 pass 462