Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

packaging the files to be installed with pip/easy_install

  • Loading branch information...
commit 9c1f1e97a81029db3903b84c769f91e4035f7c20 2 parents 2fb34b9 + 4bcb2d8
@terencehonles authored
View
14 .gitignore
@@ -1,17 +1,11 @@
*.py[oc]
*~
*.swp
-/bin/easy_install
-/bin/easy_install-2.7
-/bin/pip
-/bin/pip-2.7
+/bin
/include
/lib
/lib64
/system
-/bin/activate
-/bin/activate.csh
-/bin/activate.fish
-/bin/activate_this.py
-/bin/python
-/bin/xattr
+/build
+/dist
+/xzip.egg-info
View
1  README
View
57 README.rst
@@ -0,0 +1,57 @@
+E[x]ploded Zip File System
+==========================
+
+``xzip`` is a FUSE_ file system for deduplicating zip files which share zipped
+contents.
+
+To install use::
+
+ $ pip install xzip
+
+ or
+
+ $ easy_install xzip
+
+``xzip`` provides three executables ``zipexplode``, ``zipanalyze``, and
+``mount.xzip`` which will "explode" a zip, analyze a zip file, and mount
+exploded zips respectively.
+
+The file structure for an exploded zip is the following::
+
+ $ zipexplode path/to/zip/name-of-zip.zip
+ $ tree .
+ .
+ ├── data
+ │   ├── <sha1-data-file1>
+ │   ├── <sha1-data-file2>
+ │   ├── ...
+ │   └── <sha1-data-filen>
+ └── meta
+    ├── name-of-zip.zip.dir
+    ├── name-of-zip.zip.jump
+    └── name-of-zip.zip.stream
+
+
+Data files may be shared between an arbitrary number of exploded zips files,
+and the meta tuple (``*.dir``, ``*.jump``, ``*.stream``) describe the original
+zip file.
+
+
+``zipexplode`` accepts two options ``--directory`` and ``--depth`` to modify
+where it creates the ``data`` and ``meta`` directories and how many levels deep
+the ``data`` directory should be. ``zipexplode`` can explode multiple zip files
+at once, and additional help is provided with the ``--help`` option.
+
+
+``zipanalyze`` simply prints out the sha1 of different segments of the original
+zip file. This script was used to determine what could be deduplicated, and
+what needed to be stored per zip file. This executable is mainly of historical
+use.
+
+``mount.xzip`` will mount the directory structure described above, and needs to
+be supplied with matching ``--directory`` and ``--depth`` arguments to when
+``zipexplode`` was called. Additional arguments ``--debug``, ``--background``,
+and ``--single-threaded`` which are passed to FUSE_ and control underlying
+functionality. For more information see the ``--help`` for ``mount.xzip``.
+
+.. _FUSE: http://fuse.sourceforge.net/
View
60 setup.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+from setuptools import setup, find_packages
+
+try:
+ from lib2to3 import refactor
+ fixers = set(refactor.get_fixers_from_package('lib2to3.fixes'))
+except ImportError:
+ fixers = set()
+
+with open('README') as readme:
+ documentation = readme.read()
+
+setup(
+ name = 'xzip',
+ version = '0.9',
+ packages = find_packages(),
+
+ install_requires = ['fusepy>=1.1'],
+
+ author = 'Terence Honles',
+ author_email = 'terence@honles.com',
+ description = 'E[x]ploded zip file system in FUSE',
+ long_description = documentation,
+ license = 'PSF',
+ keywords = 'FS FileSystem File System Zip Deduplication',
+
+ entry_points = {
+ 'console_scripts': [
+ 'zipexplode = xzip.explode:main',
+ 'zipanaylze = xzip.anaylze:main',
+ 'mount.xzip = xzip.fs:main',
+ ],
+ },
+
+
+ use_2to3 = True,
+ # only use the following fixers (everything else is already compatible)
+ use_2to3_exclude_fixers = fixers - set([
+ 'lib2to3.fixes.fix_except',
+ 'lib2to3.fixes.fix_future',
+ 'lib2to3.fixes.fix_numliterals',
+ 'lib2to3.fixes.fix_reduce',
+ ]),
+
+ classifiers = [
+ 'Development Status :: 4 - Beta',
+ 'Environment :: Console',
+ 'Intended Audience :: Information Technology',
+ 'Intended Audience :: System Administrators',
+ 'License :: OSI Approved :: Python Software Foundation License',
+ 'Operating System :: MacOS',
+ 'Operating System :: POSIX',
+ 'Operating System :: Unix',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 3',
+ 'Topic :: System :: Filesystems',
+ ]
+)
View
0  xzip/__init__.py
No changes.
View
8 bin/analyze.py → xzip/analyze.py
@@ -9,6 +9,9 @@
from collections import namedtuple
from hashlib import sha1
+__all__ = ('CENTRAL_DIR', 'END_OF_DIR', 'LOCAL_HEADER', 'DATA_DESCRIPTOR',
+ 'process_zip')
+
class _Struct(struct.Struct):
__slots__ = ('marker', '_named_ctor')
@@ -118,5 +121,8 @@ def process_file(file, info):
sha1(decompressed).hexdigest())
-if __name__ == '__main__':
+def main():
process_zip(sys.argv[1])
+
+if __name__ == '__main__':
+ main()
View
8 bin/explode.py → xzip/explode.py
@@ -11,6 +11,8 @@
from hashlib import sha1
from os import path
+__all__ = ('CENTRAL_DIR', 'END_OF_DIR', 'LOCAL_HEADER', 'DATA_DESCRIPTOR',
+ 'STREAM_ITEM', 'JUMP_ITEM', 'parser', 'process_zip', 'process_file')
class _Struct(struct.Struct):
__slots__ = ('marker', '_named_ctor')
@@ -164,8 +166,12 @@ def process_file(file, info, stream, depth=0, base='.'):
parser.add_argument('filenames', metavar='FILE', nargs='+',
help='zip files to process')
-if __name__ == '__main__':
+
+def main():
args = parser.parse_args()
for filename in args.filenames:
process_zip(filename, depth=args.depth, base=args.directory)
+
+if __name__ == '__main__':
+ main()
View
12 fs.py → xzip/fs.py
@@ -16,6 +16,10 @@
from os import path
from struct import Struct
+__all__ = ('ZIP_STREAM_ITEM', 'DESCRIPTOR', 'STREAM_ITEM', 'JUMP_ITEM',
+ 'HEADER_DIFF', 'Descriptor', 'ExplodedInfo', 'ExplodedZip', 'File',
+ 'StreamItem', 'SeekTree', 'parser')
+
ZIP_STREAM_ITEM = Struct('<4s5H3L2H')
DESCRIPTOR = Struct('<3L')
STREAM_ITEM = Struct('<4s5H3L2HB20s')
@@ -340,7 +344,8 @@ def _load_stream_item(self):
header = StreamItem._make(STREAM_ITEM.unpack(raw_header))
var_fields = header.filename_len + header.extra_field_len
- sha1 = b2a_hex(header.sha)
+ # I would think that b2a_hex should decode the raw bytes...
+ sha1 = b2a_hex(header.sha).decode('ascii')
# only save the zip part of the header
self.zip_header = (raw_header[:HEADER_DIFF] +
@@ -594,9 +599,12 @@ def writeable(self):
parser.add_argument('mount', help='mount point')
-if __name__ == '__main__':
+def main():
args = parser.parse_args()
fuse = FUSE(ExplodedZip(base=args.directory, depth=args.depth),
args.mount, foreground=not args.background, ro=True,
debug=args.debug, nothreads=args.single_threaded)
+
+if __name__ == '__main__':
+ main()
Please sign in to comment.
Something went wrong with that request. Please try again.