Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Added zip info extractor.

  • Loading branch information...
commit 13eb2fffa05a19caef5273bf4e3a8ad46e441e4d 1 parent 3b7a43f
Richard authored
View
4 trunk/prodRoot/consoleWin.py
@@ -10,8 +10,8 @@
#'collectionMonitorNonRealtimeNotifierServiceV2',
#'syncXmlRpcServerV2,
'tubeDelayService',
- 'monitorService',
- 'fileListHandler'
+ 'monitorServiceV2',
+ 'folderScanner'
]
def main():
View
6 trunk/prodRoot/desktopApp/lib/compress/zipClass.py
@@ -64,6 +64,9 @@ def extract_to(self, path):
self.extract(p, path)
def extract(self, filename, path):
+ '''
+ Extract filename to path. Target file would be path/filename
+ '''
if not filename.endswith('/'):
f = os.path.join(path, filename)
dir = os.path.dirname(f)
@@ -72,6 +75,9 @@ def extract(self, filename, path):
#print 'extracting:', f
zipFilename = encode2Local(filename)
file(f, 'wb').write(self.zfile.read(zipFilename))
+ return f
+
+ raise "Invalid filename"
def list(self):
for i in self.zfile.namelist():
yield decode2Local(i)
View
2  trunk/prodRoot/localLibs/services/fileListHandler.py
@@ -44,7 +44,7 @@ def __init__(self, storage, zipCollectionRoot = gZipCollectionRoot, passwd = "12
def processJob(self, job, item):
if not (item['monitoringPath'] in self.monitoringList):
self.monitoringList.append(item['monitoringPath'])
- info = self.storage.addItem(item["fullPath"])
+ info = self.storage.addItem(item["fullPath"])
#print "zipped size", info.compress_size
self.curStorageSize += info.compress_size
View
2  trunk/prodRoot/localLibs/services/monitorServiceV2.py
@@ -54,7 +54,7 @@ def processCmd(self, job, item):
targetTube = item["targetTube"]
if not os.path.exists(fullPath) or self.notifyThreads.has_key(fullPath):
job.delete()
- t = changeNotifyForBeanstalkd(fullPath, blackList, targetTube)
+ t = changeNotifyForBeanstalkd(fullPath, targetTube, blackList)
self.notifyThreads[fullPath] = t
t.start()
View
5 trunk/prodRoot/localLibs/services/tests/folderScannerTest.py
@@ -5,7 +5,10 @@
'''
import localLibSys
import localLibs.services.folderScanner as folderScanner
-
+'''
+fileScanner->fileListTube
+monitorServiceV2->fileListTube
+'''
if __name__ == "__main__":
s = folderScanner.folderScanner()
s.addItem({"command": "folderScanner", "fullPath":"d:/tmp/monitoring",
View
10 trunk/prodRoot/localLibs/services/tests/monitorServiceTest.py
@@ -4,8 +4,12 @@
@author: Richard
'''
import localLibSys
-import localLibs.services.monitorService as monitorService
-
+import localLibs.services.monitorServiceV2 as monitorService
+'''
+fileScanner->fileListTube
+monitorServiceV2->fileListTube
+'''
if __name__ == "__main__":
s = monitorService.monitorService()
- s.addItem("d:/tmp/monitoring")
+ s.addItem({"command": "folderScanner", "fullPath":"d:/tmp/monitoring",
+ "targetTubeName": "fileListTube","blackList":[]})
View
22 trunk/prodRoot/localLibs/services/tests/zippedCollectionListHandlerTest.py
@@ -0,0 +1,22 @@
+'''
+Created on 2012-02-20
+
+@author: Richard
+'''
+import localLibSys
+import localLibs.services.folderScanner as folderScanner
+import localLibs.services.monitorServiceV2 as monitorService
+'''
+fileScanner->fileListTube
+monitorServiceV2->fileListTube
+'''
+zippedListTubeName = "zippedListTube"
+
+if __name__ == "__main__":
+ s = folderScanner.folderScanner()
+ s.addItem({"command": "folderScanner", "fullPath":"d:/tmp/target",
+ "targetTubeName": zippedListTubeName,"blackList":[]})
+
+ s = monitorService.monitorService()
+ s.addItem({"command": "folderScanner", "fullPath":"d:/tmp/target",
+ "targetTubeName": zippedListTubeName,"blackList":[]})
View
69 trunk/prodRoot/localLibs/services/zippedCollectionListHandler.py
@@ -0,0 +1,69 @@
+'''
+Created on 2012-02-13
+
+@author: Richard
+'''
+import beanstalkc
+import os
+
+#from pprint import pprint
+
+import localLibSys
+from localLibs.storage.infoStorage.zippedCollectionWithInfo import zippedCollectionWithInfo
+from localLibs.localFs.tmpFile import getStorgePathWithDateFolder
+import desktopApp.lib.archiver.encryptionStorageBase as encryptionStorageBase
+from fileListHandlerBase import fileListHandlerBase
+import wwjufsdatabase.libs.utils.transform as transform
+import localLibs.collection.collectionDatabaseV2 as collectionDatabase
+from localLibs.objSys.objectDatabaseV3 import objectDatabase
+from localLibs.storage.infoStorage.zippedInfo import zippedInfo
+import wwjufsdatabase.libs.utils.simplejson as json
+
+gBeanstalkdServerHost = '127.0.0.1'
+gBeanstalkdServerPort = 11300
+gMonitorServiceTubeName = "monitorQueue"
+gFileListTubeName = "fileListDelayed"
+
+gMaxZippedCollectionSize = 0.5*1024
+
+gZipCollectionRoot = "d:/tmp/generating"
+gWorkingDir = "d:/tmp/working"
+
+class zippedCollectionListHandler(fileListHandlerBase):
+ '''
+ classdocs
+ '''
+ def __init__(self, fileListTubeName = gFileListTubeName, passwd = "123", workingDir = gWorkingDir):
+ '''
+ Constructor
+ '''
+ self.encCopier = encryptionStorageBase.arc4EncSimpleCopier(passwd)
+ self.decCopier = encryptionStorageBase.arc4DecSimpleCopier(passwd)
+ self.collectionDict = {}
+ self.workingDir = workingDir
+ self.dbInst = objectDatabase()
+ super(zippedCollectionListHandler, self).__init__(fileListTubeName)
+
+ def processJob(self, job, item):
+ monitoringFullPath = transform.transformDirToInternal(item['monitoringPath'])
+ archiveId = "zippedInfoColllection://" + monitoringFullPath
+ if not self.collectionDict.has_key(monitoringFullPath):
+ self.collectionDict[monitoringFullPath] = collectionDatabase.collectionOnMongoDbBase(archiveId, self.dbInst.getCollectionDb())
+ #Save the item in the archive collection: zippedInfoColllection://D:/tmp/
+ fullPath = transform.transformDirToInternal(item["fullPath"])
+ relativePath = transform.getRelativePathFromFull(fullPath, monitoringFullPath)
+ if not self.collectionDict[monitoringFullPath].exists(relativePath):
+ #This item is not in the collection, so we need to extract info from this item
+ newObj = self.dbInst.getFsObjFromFullPath(fullPath)
+ self.collectionDict[monitoringFullPath].addObj(relativePath, newObj["uuid"])
+ for i in zippedInfo(self.workingDir).enumItems(fullPath):
+ fp = open(i, 'r')
+ loadedFileInfo = json.load(fp)
+ print loadedFileInfo
+ return True
+
+if __name__ == "__main__":
+ print 'starting fileListHandler'
+ workingDir = "d:/tmp/working"
+ s = zippedCollectionListHandler()
+ s.startServer()
View
1  trunk/prodRoot/localLibs/storage/infoStorage/zippedCollectionWithInfo.py
@@ -1,5 +1,4 @@
import localLibSys
-import wwjufsdatabase.libs.utils.simplejson as json
import wwjufsdatabase.libs.utils.transform as transform
import localLibs.objSys.ufsObj as ufsObj
import wwjufsdatabase.libs.utils.fileTools as fileTools
View
30 trunk/prodRoot/localLibs/storage/infoStorage/zippedInfo.py
@@ -1,3 +1,4 @@
+import re
import localLibSys
import wwjufsdatabase.libs.utils.simplejson as json
import wwjufsdatabase.libs.utils.transform as transform
@@ -7,6 +8,8 @@
gWorkingDir = "d:/tmp"
gDefaultInfoSize = 100
+gInfoFilePrefix = 'zippedCollFile'
+gInfoFileExt = ".log"
class zippedInfo(object):
def __init__(self, workingDir = gWorkingDir):
@@ -27,18 +30,13 @@ def addItem(self, fullPath):
#return self.getZipFile().addfile(unicode(fullPath), unicode(fullPath))
return gDefaultInfoSize
- def getZipFile(self):
- if self.zipFile is None:
- self.zipFilePath = transform.transformDirToInternal(
- fileTools.getTimestampWithFreeName(self.workingDir, '.zip'))
- self.zipFile = zipClass.ZFile(self.zipFilePath, 'w')
- return self.zipFile
+
def finalizeZipFile(self):
#Add info to zip file
self.additionalInfoDict["collectionContentInfo"] = self.collectionInfoDict
s = json.dumps(self.additionalInfoDict, sort_keys=True, indent=4)
infoFilePath = transform.transformDirToInternal(
- fileTools.getTimestampWithFreeName(self.workingDir, '.log', 'zippedCollFile'))
+ fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileExt, gInfoFilePrefix))
logFile = open(infoFilePath, 'w')
logFile.write(s)
logFile.close()
@@ -46,4 +44,20 @@ def finalizeZipFile(self):
self.zipFile.close()
#Set attribute so new zip will be created if this object is still in use
self.zipFile = None
- return self.zipFilePath
+ return self.zipFilePath
+
+ def enumItems(self, archiveFullPath):
+ zipFile = zipClass.ZFile(self.zipFilePath, 'r')
+ for i in zipFile.list():
+ if not (re.search("^"+gInfoFilePrefix + ".*" + gInfoFileExt + "$", i) is None):
+ yield zipFile.extract(i, self.workingDir)
+
+
+ ################################################
+ # The following functions are not recommended to be called from outside of this class
+ def getZipFile(self):
+ if self.zipFile is None:
+ self.zipFilePath = transform.transformDirToInternal(
+ fileTools.getTimestampWithFreeName(self.workingDir, '.zip'))
+ self.zipFile = zipClass.ZFile(self.zipFilePath, 'w')
+ return self.zipFile
Please sign in to comment.
Something went wrong with that request. Please try again.