From 64028c529bf8213f9f6906264eda8976e09c5218 Mon Sep 17 00:00:00 2001 From: Roshan Sumbaly Date: Wed, 6 Oct 2010 10:39:50 -0700 Subject: [PATCH] Fixed read-only store grandfathering script to now output metadata --- bin/grandfather-readonly.sh | 26 ++++++++++---- .../gui/ReadOnlyStoreManagementServlet.java | 15 +++++--- .../admin/AdminServiceRequestHandler.java | 35 +++++++++++++------ ...etchPartitionFileStreamRequestHandler.java | 5 +-- 4 files changed, 58 insertions(+), 23 deletions(-) diff --git a/bin/grandfather-readonly.sh b/bin/grandfather-readonly.sh index c5541d1b1f..66aa9ef992 100755 --- a/bin/grandfather-readonly.sh +++ b/bin/grandfather-readonly.sh @@ -15,29 +15,41 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# One time grandfathering script to convert all read-only store dirs to +# new format if [ $# -ne 1 ]; then echo 'USAGE: bin/grandfather-readonly.sh [readonly-folder]' exit 1 fi -# Read args +# Read args READ_ONLY_DIR=$1 +# Create temporary metadata file +METADATA_FILE="/tmp/$(basename $0).$$.tmp" +echo "{\"format\":\"ro0\"}" > $METADATA_FILE + for stores in $READ_ONLY_DIR/* do if [ -d $stores ]; then + echo ---Working on store ${stores} --- # Convert all to .temp numVersions=`find $stores -name version-* | grep -v .bak | grep -v .temp | wc -l` + maxVersion=`find $stores -name version-* | grep -v .bak | grep -v .temp | awk -F'-' '{print $2}' | sort -n | tail -1` + if [ $numVersions -eq 1 ]; then + cp $METADATA_FILE ${stores}/version-${maxVersion}/.metadata + echo Added metadata to ${stores}/version-${maxVersion} + fi if [ $numVersions -gt 1 ]; then - maxVersion=`find $stores -name version-* | grep -v .bak | grep -v .temp | awk -F'-' '{print $2}' | sort -n | tail -1` for versionDirNo in `find $stores -name version-* | grep -v .bak | grep -v .temp | awk -F'-' '{print $2}' | sort -n` do - echo $stores/version-$versionDirNo $stores/version-$maxVersion.temp - mv $stores/version-$versionDirNo $stores/version-$maxVersion.temp + mv ${stores}/version-${versionDirNo} ${stores}/version-${maxVersion}.temp + echo Moved ${stores}/version-${versionDirNo} to ${stores}/version-${maxVersion}.temp + cp $METADATA_FILE ${stores}/version-${maxVersion}.temp/.metadata + echo Added metadata to ${stores}/version-${maxVersion}.temp let maxVersion=maxVersion-1 done - echo '-------' fi # Convert all .temp to normal @@ -45,9 +57,9 @@ do if [ $numVersionsTmp -gt 1 ]; then for versionDir in `find $stores -name version-*.temp | grep -v .bak | awk -F'.' '{print $1}'` do - mv $versionDir.temp $versionDir + mv ${versionDir}.temp ${versionDir} + echo Moved ${versionDir}.temp to ${versionDir} done - echo '=======' fi fi done diff --git a/src/java/voldemort/server/http/gui/ReadOnlyStoreManagementServlet.java b/src/java/voldemort/server/http/gui/ReadOnlyStoreManagementServlet.java index f646320803..84901ea40b 100644 --- a/src/java/voldemort/server/http/gui/ReadOnlyStoreManagementServlet.java +++ b/src/java/voldemort/server/http/gui/ReadOnlyStoreManagementServlet.java @@ -132,6 +132,7 @@ private List getReadOnlyStores(VoldemortServer server) { @Override public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + initStores((VoldemortServer) getServletContext().getAttribute(VoldemortServletContextListener.SERVER_KEY)); Map params = Maps.newHashMap(); params.put("stores", stores); velocityEngine.render("read-only-mgmt.vm", params, resp.getOutputStream()); @@ -162,7 +163,6 @@ private void doSwap(HttpServletRequest req, HttpServletResponse resp) throws IOE ServletException { String dir = getRequired(req, "dir"); String storeName = getRequired(req, "store"); - String formatString = getOptional(req, "format"); ReadOnlyStorageEngine store = this.getStore(storeName); if(store == null) @@ -200,15 +200,21 @@ private void doFetch(HttpServletRequest req, HttpServletResponse resp) throws IO File fetchDir = null; if(fileFetcher == null) { - logger.warn("File fetcher class has not instantiated correctly"); - fetchDir = new File(fetchUrl); + logger.warn("File fetcher class has not instantiated correctly. Assuming local file"); + + if(!Utils.isReadableDir(fetchUrl)) { + throw new VoldemortException("Fetch url " + fetchUrl + " is not readable"); + } + + fetchDir = new File(store.getStoreDirPath(), "version-" + Long.toString(pushVersion)); + Utils.move(new File(fetchUrl), fetchDir); } else { logger.info("Executing fetch of " + fetchUrl); try { fetchDir = fileFetcher.fetch(fetchUrl, store.getStoreDirPath() + File.separator - + "version-" + pushVersion); + + "version-" + Long.toString(pushVersion)); if(fetchDir == null) { throw new ServletException("File fetcher failed for " + fetchUrl @@ -255,6 +261,7 @@ private String getRequired(HttpServletRequest req, String name) throws ServletEx } private ReadOnlyStorageEngine getStore(String storeName) throws ServletException { + initStores((VoldemortServer) getServletContext().getAttribute(VoldemortServletContextListener.SERVER_KEY)); for(ReadOnlyStorageEngine store: this.stores) if(store.getName().equals(storeName)) return store; diff --git a/src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java b/src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java index fb31729833..5950c742b7 100644 --- a/src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java +++ b/src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java @@ -237,8 +237,9 @@ public VAdminProto.GetROCurrentVersionDirResponse handleGetROCurrentVersionDir(V try { for(String storeName: storeNames) { - ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository, - storeName); + ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore, + storeRepository, + storeName); VAdminProto.ROStoreVersionDirMap storeResponse = VAdminProto.ROStoreVersionDirMap.newBuilder() .setStoreName(storeName) .setStoreDir(store.getCurrentDirPath()) @@ -260,8 +261,9 @@ public VAdminProto.GetROMaxVersionDirResponse handleGetROMaxVersionDir(VAdminPro try { for(String storeName: storeNames) { - ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository, - storeName); + ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore, + storeRepository, + storeName); File storeDirPath = new File(store.getStoreDirPath()); if(!storeDirPath.exists()) @@ -406,8 +408,9 @@ public VAdminProto.RollbackStoreResponse handleRollbackStore(VAdminProto.Rollbac VAdminProto.RollbackStoreResponse.Builder response = VAdminProto.RollbackStoreResponse.newBuilder(); try { - ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository, - storeName); + ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore, + storeRepository, + storeName); File rollbackVersionDir = new File(store.getStoreDirPath(), "version-" + pushVersion); @@ -446,8 +449,10 @@ public VAdminProto.SwapStoresAndCleanStateResponse handleSwapStoresAndCleanState } private void swapStore(String storeName, String directory) throws VoldemortException { - ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository, - storeName); + + ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore, + storeRepository, + storeName); if(!Utils.isReadableDir(directory)) throw new VoldemortException("Store directory '" + directory @@ -488,8 +493,9 @@ public VAdminProto.AsyncOperationStatusResponse handleFetchStore(VAdminProto.Fet .setDescription("Fetch store") .setStatus("started"); try { - final ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository, - storeName); + final ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore, + storeRepository, + storeName); final long pushVersion; if(request.hasPushVersion()) { pushVersion = request.getPushVersion(); @@ -979,6 +985,15 @@ static int valueSize(Versioned value) { return value.getValue().length + ((VectorClock) value.getVersion()).sizeInBytes() + 1; } + static ReadOnlyStorageEngine getReadOnlyStorageEngine(MetadataStore metadata, + StoreRepository repo, + String name) { + if(metadata.getStoreDef(name).getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) != 0) + throw new VoldemortException("Store " + name + + " is not a read-only store, cannot complete operation"); + return (ReadOnlyStorageEngine) getStorageEngine(repo, name); + } + static StorageEngine getStorageEngine(StoreRepository storeRepository, String storeName) { StorageEngine storageEngine = storeRepository.getStorageEngine(storeName); diff --git a/src/java/voldemort/server/protocol/admin/FetchPartitionFileStreamRequestHandler.java b/src/java/voldemort/server/protocol/admin/FetchPartitionFileStreamRequestHandler.java index 8276bbcb3d..fd55360243 100644 --- a/src/java/voldemort/server/protocol/admin/FetchPartitionFileStreamRequestHandler.java +++ b/src/java/voldemort/server/protocol/admin/FetchPartitionFileStreamRequestHandler.java @@ -49,8 +49,9 @@ protected FetchPartitionFileStreamRequestHandler(VAdminProto.FetchPartitionFiles throw new VoldemortException("Should be fetching partition files only for read-only stores"); } - this.storageEngine = (ReadOnlyStorageEngine) AdminServiceRequestHandler.getStorageEngine(storeRepository, - request.getStore()); + this.storageEngine = AdminServiceRequestHandler.getReadOnlyStorageEngine(metadataStore, + storeRepository, + request.getStore()); this.blockSize = voldemortConfig.getAllProps() .getLong("partition.buffer.size.bytes",