Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

add migrate tool

  • Loading branch information...
commit e40bc6b6f0f2846e592dbb51fa27d267421029cc 1 parent 02067aa
@liut authored
View
17 imsto/_util.py
@@ -14,7 +14,7 @@
__all__ = [
'check_dirs', 'save_file', 'guessImageType',
'thumbnail_wand', 'thumb_image', 'watermark_image',
-'guess_mimetype', 'password_hash',
+'guess_mimetype', 'guess_ext', 'password_hash',
'encode_upload'
]
@@ -154,6 +154,13 @@ def guess_mimetype(fn, default="application/octet-stream"):
if ext == "jpg": ext = "jpeg"
return mimetypes.guess_type(bfn + "." + ext)[0] or default
+def guess_ext(mime):
+ import mimetypes
+ ext = mimetypes.guess_extension(mime)
+ if ext == '.jpe':
+ return '.jpg'
+ return ext
+
def watermark_image(filename, distname):
from image import SimpImage
im = SimpImage(filename)
@@ -191,6 +198,7 @@ def encode_upload(file=None, content=None, content_type=None, name=None, ext_dat
"""
BOUNDARY = '----------bundary------'
CRLF = '\r\n'
+ #print CRLF
body = []
# Add the metadata about the upload first
for key, value in ext_data:
@@ -212,6 +220,8 @@ def encode_upload(file=None, content=None, content_type=None, name=None, ext_dat
content = f.read()
f.close()
+ #print 'type content: %s, len content: %s' % (type(content), len(content))
+
if name is None:
ext = guessImageType(content[:32])
name = 'data.{}'.format(ext)
@@ -221,10 +231,9 @@ def encode_upload(file=None, content=None, content_type=None, name=None, ext_dat
body.extend(
['--' + BOUNDARY,
- 'Content-Disposition: form-data; name="file"; filename="%s"'
- % name,
+ str('Content-Disposition: form-data; name="file"; filename="%s"' % name),
# The upload server determines the mime-type, no need to set it.
- 'Content-Type: %s' % content_type,
+ str('Content-Type: %s' % content_type),
'',
content,
])
View
57 imsto/migrate.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+tool.py
+
+Created by liut on 2013-06-18.
+Copyright (c) 2010-2013 liut. All rights reserved.
+"""
+
+import sys
+import os
+from store import load_imsto, Config, encode_upload
+
+def test_section(section):
+ config = Config()
+ return config.has_section(section)
+
+def migrate(from_section,to_section):
+ """merge and sync data between 2 storage engines"""
+ if from_section == to_section:
+ return False
+
+ if not test_section(from_section) or not test_section(to_section):
+ return False
+
+ imsto1 = load_imsto(from_section)
+ imsto2 = load_imsto(to_section)
+
+ total = imsto1.count()
+ print 'total: {}'.format(total)
+
+ limit = 50
+ offset = 0
+ while offset < total:
+ print 'migrating {}/{}'.format(offset, total)
+ for item in imsto1.browse(limit, offset, only_items=True):
+ #print item
+ _store_item(imsto2, item)
+ offset += limit
+
+def _store_item(imsto2, item):
+ print 'item size: %s' % item.size
+ r = imsto2.store(item.file, ctype=item.mime, name=item.name)
+ print 'store %s (%s): %s' % (item.id, item.name, r)
+
+if __name__ == '__main__':
+ import argparse
+ parser = argparse.ArgumentParser(usage='%(prog)s command')
+ parser.add_argument('-F','--src',metavar='section',type=str)
+ parser.add_argument('-T','--dst',metavar='section',type=str)
+ args, remaining = parser.parse_known_args()
+
+ #migrate('imsto', 'weed')
+ if args.src and args.dst:
+ migrate(args.src, args.dst)
+ else:
+ parser.print_help()
View
128 imsto/store.py
@@ -59,7 +59,7 @@ def browse(self, limit=20, start=0, sort=None, only_items = False):
sort = [('uploadDate',DESCENDING)]
cursor = self.collection.find(limit=limit,skip=start,sort=sort)
- items = [_make_item(item) for item in cursor]
+ items = [StoreItem(self, item) for item in cursor]
if only_items:
return items
url_prefix = urljoin(self.get_config('url_prefix'), self.get_config('thumb_path'))
@@ -73,11 +73,11 @@ def count(self):
# return self
# def next(self):
# if self.__cursor:
- # return _make_item(self.__cursor.next())
+ # return StoreItem(self, self.__cursor.next())
# raise StopIteration
def store(self, file=None, ctype=None, content=None, name=None):
- """save a file-like to mongodb"""
+ """save a file-like item"""
if content is None and not hasattr(file, 'read'):
raise TypeError('invalid file-like object')
@@ -118,7 +118,9 @@ def store(self, file=None, ctype=None, content=None, name=None):
spec = {'_id': id,'filename': filename, 'hash': hashed, 'content_type': ctype, 'content_length': size}
if name:
spec['name'] = name
- return [True, self._put(data, **spec), filename]
+ rr = self._put(data, **spec)
+ if rr:
+ return [True, rr, filename]
def get_meta(self, id=None, filename=None):
spec = None
@@ -130,7 +132,7 @@ def get_meta(self, id=None, filename=None):
print 'spec %s' % spec
item = self.collection.find_one(spec)
if item:
- return _make_item(item)
+ return StoreItem(self, item)
def _save_meta(self, id, spec):
'''mongo special meta data'''
@@ -204,10 +206,11 @@ def load(self, path):
if not os.path.exists(org_file):
print('fetching file: {}'.format(org_path))
- ret = self.prepare(org_file, path=org_path, id=id)
- if ret is False:
+ file = self.fetch(id, path=org_path)
+ if file is False:
print('fetch failed')
raise UrlError('id {} not found'.format(id))
+ save_file(file, org_file)
if not os.path.exists(org_file):
raise UrlError('file not found')
@@ -255,26 +258,67 @@ def load(self, path):
return (dst_file, dst_path)
- def prepare(self, filename, path, id):
+ def fetch(self, id, path):
key = path if self.engine == 's3' else id
- file = None
try:
- file = self._get(key)
+ return self._get(key)
except Exception, e:
- self.close()
-
- if file is None:
print('prepare: {} not found'.format(key))
- return False
+ raise e
+ finally:
+ self.close()
- save_file(file, filename)
def url(self, path, size='orig'):
url_prefix = self.get_config('url_prefix')
thumb_path = self.get_config('thumb_path')
return '{}/{}/{}/{}'.format(url_prefix.rstrip('/'), thumb_path.strip('/'), size, path)
+class StoreItem(dict):
+ """docstring for Item"""
+ imsto = None
+ id = None
+ def __init__(self, imsto, meta):
+ self.imsto = imsto
+ if isinstance(meta, dict):
+ item = self._fix_meta(meta)
+ for key, value in item.iteritems():
+ self[key.lower()] = value
+ if key == 'id':
+ self.id = value
+ if not self.has_key('name') or self['name'] is None:
+ self.name = self.id + guess_ext(self['mime'])
+
+ def __getattr__(self, name):
+ if name == 'dict':
+ return self
+ else:
+ return self.get(name, None)
+
+ def _fix_meta(self, item):
+ '''convert mongo item to simple'''
+ item['id'] = item.pop('_id')
+ if item.has_key('length'):
+ item['size'] = item.pop('length')
+ elif item.has_key('content_length'):
+ item['size'] = item.pop('content_length')
+ if item.has_key('uploadDate'):
+ item['created'] = item.pop('uploadDate')
+ if item.has_key('contentType'):
+ item['mime'] = item.pop('contentType')
+ if item.has_key('content_type'):
+ item['mime'] = item.pop('content_type')
+ if not item.has_key('filename') and item.has_key('path'):
+ item['filename'] = item.pop('path')
+ item.pop('chunkSize', None)
+ item.pop('app_id', None)
+ # print item
+ return item
+
+ @property
+ def file(self):
+ return self.imsto._get(self)
class EngineError(Exception):
@@ -304,25 +348,6 @@ def _make_id(hashed, size=None):
raise TypeError('expected a int, not ' + repr(size))
return base_convert('{}{:02x}'.format(hashed, size % 255), 16, 36)
-def _make_item(item):
- '''convert mongo item to simple'''
- newItem = item.copy()
- newItem['id'] = newItem.pop('_id')
- if newItem.has_key('length'):
- newItem['size'] = newItem.pop('length')
- elif newItem.has_key('content_length'):
- newItem['size'] = newItem.pop('content_length')
- if newItem.has_key('uploadDate'):
- newItem['created'] = newItem.pop('uploadDate')
- if newItem.has_key('contentType'):
- newItem['mime'] = newItem.pop('contentType')
- if not newItem.has_key('filename') and newItem.has_key('path'):
- newItem['filename'] = newItem.pop('path')
- newItem.pop('chunkSize', None)
- newItem.pop('app_id', None)
- # print newItem
- return newItem
-
class StoreEngineGridFs(StoreBase):
@@ -332,12 +357,9 @@ class StoreEngineGridFs(StoreBase):
def __init__(self, section):
StoreBase.__init__(self, section)
- def _get(self, id=None, filename=None):
- """retrieve a file by id"""
- if filename:
- item = self.get_meta(filename=filename)
- if item:
- id = item.id
+ def _get(self, id):
+ if isinstance(id, StoreItem):
+ id = id.id
if id and self.exists(id):
return self.fs.get(id)
@@ -364,15 +386,13 @@ def fs(self):
class StoreEngineS3(StoreBase):
"""docstring for StoreEngineS3"""
+ _bucket = None
def __init__(self, section):
StoreBase.__init__(self, section)
- self.bucket = self.get_config('bucket_name')
- self.AccessKey = self.get_config('s3_access_key')
- self.SecretKey = self.get_config('s3_secret_key')
- def _get(self, key):
- s = self.get_s3_bucket()
- return s.get(key)
+ def _get(self, id):
+ item = id if isinstance(id, StoreItem) else self.get_meta(id)
+ return self.bucket.get(key)
def delete(self, key):
raise NotImplemented()
@@ -386,10 +406,15 @@ def _put(self, data, filename, content_type):
def _store_exists(self, id=None, *args, **kwargs):
raise NotImplemented()
- def get_s3_bucket(self):
- from simples3 import S3Bucket, KeyNotFound
- s = S3Bucket(self.bucket, access_key=self.AccessKey, secret_key=self.SecretKey)
- return s
+ @property
+ def bucket(self):
+ if self._bucket is None:
+ from simples3 import S3Bucket, KeyNotFound
+ bucket_name = self.get_config('bucket_name')
+ access_key = self.get_config('s3_access_key')
+ secret_key = self.get_config('s3_secret_key')
+ self._bucket = S3Bucket(bucket_name, access_key=access_key, secret_key=secret_key)
+ return self._bucket
WEED_HOST = 'weed_vol_host'
WEED_FID = 'weed_fid'
@@ -403,7 +428,7 @@ def __init__(self, section):
def _get(self, id):
print '_get {}'.format(id)
- item = self.get_meta(id)
+ item = id if isinstance(id, StoreItem) else self.get_meta(id)
if not item.has_key(WEED_HOST) or not item.has_key(WEED_HOST):
raise ValueError('the entry has no special value ' + WEED_HOST + ' and ' + WEED_FID)
volume_host, fid = item[WEED_HOST], item[WEED_FID]
@@ -426,6 +451,7 @@ def _put(self, data, **spec):
spec[WEED_FID] = fid
self._save_meta(spec['_id'], spec)
return spec['_id']
+ print 'store error: %s' % ret
def _store_exists(self, id=None, *args, **kwargs):
View
7 imsto/tool.py
@@ -23,9 +23,10 @@ def list_dir(limit=5,start=0,prefix=''):
return
print 'total {}'.format(gallery['total'])
- for img in gallery['items']:
- #print(img)
- print("{0[filename]}\t{0[size]:8,d}".format(img))
+ for item in gallery['items']:
+ #print hasattr(item.file, 'read')
+ #print(item)
+ print("{0[filename]}\t{0[size]:8,d}".format(item))
def store_file(filename):
if os.access(filename, os.R_OK):
View
11 imsto/weedfs.py
@@ -54,7 +54,7 @@ def assign(self, count = 1, replication = None):
#{"count":1,"fid":"2,753b19e78fe6","publicUrl":"127.0.0.1:9334","url":"localhost:9334"}
first, second, result = self._request(url)
#volume_host, fid =
- if isinstance(result, dict):
+ if first >= 200 and first < 300 and isinstance(result, dict):
return result['publicUrl'], result['fid']
print 'error assign {}: {}, content: {}'.format(first, second, result)
@@ -89,10 +89,11 @@ def store(self, volume_host, fid, file = None, content = None, name = None, cont
content_type, body = encode_upload(file=file, content=content, name=name, content_type=content_type)
headers = { 'Content-Type': content_type }
url = 'http://{}/{}'.format(volume_host, fid)
- ret = self._request(url, 'POST', body=body, headers=headers)
- print type(ret)
- if isinstance(ret, dict):
- return ret['size']
+ first, second, result = self._request(url, 'POST', body=body, headers=headers)
+ print type(result)
+ print result
+ if isinstance(result, dict):
+ return result['size']
View
2  static/admin/gallery.htm
@@ -85,7 +85,7 @@
alt += '上传时间: ' + item.created;
}
var _li = $("<li></li>").attr("id", "li_iid" + item.id).appendTo("#image-list"),
- _a = $("<a></a>").attr("href", url_prefix + item.filename).attr("title", title+" ( "+alt+" )").attr('rel','box'),
+ _a = $("<a></a>").attr("href", url_prefix + 'orig/' + item.filename).attr("title", title+" ( "+alt+" )").attr('rel','box'),
_img = $("<img/>").attr("src", url_prefix + 's120/' + item.filename).attr('alt',alt).attr("class", ""),
_txt = $("<span></span>").attr("class", "lbl").attr("title",item.size).text(format_size(item.size)),
_btn = $("<span></span>").attr("class", "btn btn_delete ui-corner-all ui-icon ui-icon-trash").attr("title", "删除")
Please sign in to comment.
Something went wrong with that request. Please try again.