Fixed read-only store grandfathering script to now output metadata

voldemort · Oct 6, 2010 · 64028c5 · 64028c5
1 parent 31387a8
commit 64028c5
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 23 deletions.
diff --git a/bin/grandfather-readonly.sh b/bin/grandfather-readonly.sh
@@ -15,39 +15,51 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+# One time grandfathering script to convert all read-only store dirs to 
+# new format
 if [ $# -ne 1 ];
 then
         echo 'USAGE: bin/grandfather-readonly.sh [readonly-folder]'
         exit 1
 fi
-#  Read args
+# Read args
 READ_ONLY_DIR=$1
 
+# Create temporary metadata file
+METADATA_FILE="/tmp/$(basename $0).$$.tmp"
+echo "{\"format\":\"ro0\"}" > $METADATA_FILE
+
 for stores in $READ_ONLY_DIR/*
 do
         if [ -d $stores ]; then
 
+		echo ---Working on store ${stores} ---
 		# Convert all to .temp
                 numVersions=`find $stores -name version-* | grep -v .bak | grep -v .temp | wc -l`
+                maxVersion=`find $stores -name version-* | grep -v .bak | grep -v .temp | awk -F'-' '{print $2}' | sort -n | tail -1`
+		if [ $numVersions -eq 1 ]; then
+                                cp $METADATA_FILE ${stores}/version-${maxVersion}/.metadata
+				echo Added metadata to ${stores}/version-${maxVersion}
+		fi
 		if [ $numVersions -gt 1 ]; then
-                        maxVersion=`find $stores -name version-* | grep -v .bak | grep -v .temp | awk -F'-' '{print $2}' | sort -n | tail -1`
                         for versionDirNo in `find $stores -name version-* | grep -v .bak | grep -v .temp | awk -F'-' '{print $2}' | sort -n`
                         do
-                                echo $stores/version-$versionDirNo $stores/version-$maxVersion.temp
+                                mv ${stores}/version-${versionDirNo} ${stores}/version-${maxVersion}.temp
-                                mv $stores/version-$versionDirNo $stores/version-$maxVersion.temp
+				echo Moved ${stores}/version-${versionDirNo} to ${stores}/version-${maxVersion}.temp
+                                cp $METADATA_FILE ${stores}/version-${maxVersion}.temp/.metadata
+				echo Added metadata to ${stores}/version-${maxVersion}.temp
 				let maxVersion=maxVersion-1
                         done
-			echo '-------'
                 fi
 
 		# Convert all .temp to normal
                 numVersionsTmp=`find $stores -name version-*.temp | grep -v .bak | wc -l`
         	if [ $numVersionsTmp -gt 1 ]; then
                         for versionDir in `find $stores -name version-*.temp | grep -v .bak | awk -F'.' '{print $1}'`
                         do
-                                mv $versionDir.temp $versionDir
+                                mv ${versionDir}.temp ${versionDir}
+				echo Moved ${versionDir}.temp to ${versionDir}
                         done
-                        echo '======='
                 fi
 	fi
 done

diff --git a/src/java/voldemort/server/http/gui/ReadOnlyStoreManagementServlet.java b/src/java/voldemort/server/http/gui/ReadOnlyStoreManagementServlet.java
@@ -132,6 +132,7 @@ private List<ReadOnlyStorageEngine> getReadOnlyStores(VoldemortServer server) {
     @Override
     public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException,
             IOException {
+        initStores((VoldemortServer) getServletContext().getAttribute(VoldemortServletContextListener.SERVER_KEY));
         Map<String, Object> params = Maps.newHashMap();
         params.put("stores", stores);
         velocityEngine.render("read-only-mgmt.vm", params, resp.getOutputStream());
@@ -162,7 +163,6 @@ private void doSwap(HttpServletRequest req, HttpServletResponse resp) throws IOE
             ServletException {
         String dir = getRequired(req, "dir");
         String storeName = getRequired(req, "store");
-        String formatString = getOptional(req, "format");
 
         ReadOnlyStorageEngine store = this.getStore(storeName);
         if(store == null)
@@ -200,15 +200,21 @@ private void doFetch(HttpServletRequest req, HttpServletResponse resp) throws IO
         File fetchDir = null;
         if(fileFetcher == null) {
 
-            logger.warn("File fetcher class has not instantiated correctly");
+            logger.warn("File fetcher class has not instantiated correctly. Assuming local file");
-            fetchDir = new File(fetchUrl);
+
+            if(!Utils.isReadableDir(fetchUrl)) {
+                throw new VoldemortException("Fetch url " + fetchUrl + " is not readable");
+            }
+
+            fetchDir = new File(store.getStoreDirPath(), "version-" + Long.toString(pushVersion));
+            Utils.move(new File(fetchUrl), fetchDir);
 
         } else {
             logger.info("Executing fetch of " + fetchUrl);
 
             try {
                 fetchDir = fileFetcher.fetch(fetchUrl, store.getStoreDirPath() + File.separator
-                                                       + "version-" + pushVersion);
+                                                       + "version-" + Long.toString(pushVersion));
 
                 if(fetchDir == null) {
                     throw new ServletException("File fetcher failed for " + fetchUrl
@@ -255,6 +261,7 @@ private String getRequired(HttpServletRequest req, String name) throws ServletEx
     }
 
     private ReadOnlyStorageEngine getStore(String storeName) throws ServletException {
+        initStores((VoldemortServer) getServletContext().getAttribute(VoldemortServletContextListener.SERVER_KEY));
         for(ReadOnlyStorageEngine store: this.stores)
             if(store.getName().equals(storeName))
                 return store;

diff --git a/src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java b/src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java
@@ -237,8 +237,9 @@ public VAdminProto.GetROCurrentVersionDirResponse handleGetROCurrentVersionDir(V
         try {
             for(String storeName: storeNames) {
 
-                ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
+                ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
-                                                                                       storeName);
+                                                                       storeRepository,
+                                                                       storeName);
                 VAdminProto.ROStoreVersionDirMap storeResponse = VAdminProto.ROStoreVersionDirMap.newBuilder()
                                                                                                  .setStoreName(storeName)
                                                                                                  .setStoreDir(store.getCurrentDirPath())
@@ -260,8 +261,9 @@ public VAdminProto.GetROMaxVersionDirResponse handleGetROMaxVersionDir(VAdminPro
         try {
             for(String storeName: storeNames) {
 
-                ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
+                ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
-                                                                                       storeName);
+                                                                       storeRepository,
+                                                                       storeName);
                 File storeDirPath = new File(store.getStoreDirPath());
 
                 if(!storeDirPath.exists())
@@ -406,8 +408,9 @@ public VAdminProto.RollbackStoreResponse handleRollbackStore(VAdminProto.Rollbac
         VAdminProto.RollbackStoreResponse.Builder response = VAdminProto.RollbackStoreResponse.newBuilder();
 
         try {
-            ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
+            ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
-                                                                                   storeName);
+                                                                   storeRepository,
+                                                                   storeName);
 
             File rollbackVersionDir = new File(store.getStoreDirPath(), "version-" + pushVersion);
 
@@ -446,8 +449,10 @@ public VAdminProto.SwapStoresAndCleanStateResponse handleSwapStoresAndCleanState
     }
 
     private void swapStore(String storeName, String directory) throws VoldemortException {
-        ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
+
-                                                                               storeName);
+        ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
+                                                               storeRepository,
+                                                               storeName);
 
         if(!Utils.isReadableDir(directory))
             throw new VoldemortException("Store directory '" + directory
@@ -488,8 +493,9 @@ public VAdminProto.AsyncOperationStatusResponse handleFetchStore(VAdminProto.Fet
                                                                                                             .setDescription("Fetch store")
                                                                                                             .setStatus("started");
         try {
-            final ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
+            final ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
-                                                                                         storeName);
+                                                                         storeRepository,
+                                                                         storeName);
             final long pushVersion;
             if(request.hasPushVersion()) {
                 pushVersion = request.getPushVersion();
@@ -979,6 +985,15 @@ static int valueSize(Versioned<byte[]> value) {
         return value.getValue().length + ((VectorClock) value.getVersion()).sizeInBytes() + 1;
     }
 
+    static ReadOnlyStorageEngine getReadOnlyStorageEngine(MetadataStore metadata,
+                                                          StoreRepository repo,
+                                                          String name) {
+        if(metadata.getStoreDef(name).getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) != 0)
+            throw new VoldemortException("Store " + name
+                                         + " is not a read-only store, cannot complete operation");
+        return (ReadOnlyStorageEngine) getStorageEngine(repo, name);
+    }
+
     static StorageEngine<ByteArray, byte[], byte[]> getStorageEngine(StoreRepository storeRepository,
                                                                      String storeName) {
         StorageEngine<ByteArray, byte[], byte[]> storageEngine = storeRepository.getStorageEngine(storeName);

diff --git a/src/java/voldemort/server/protocol/admin/FetchPartitionFileStreamRequestHandler.java b/src/java/voldemort/server/protocol/admin/FetchPartitionFileStreamRequestHandler.java
@@ -49,8 +49,9 @@ protected FetchPartitionFileStreamRequestHandler(VAdminProto.FetchPartitionFiles
             throw new VoldemortException("Should be fetching partition files only for read-only stores");
         }
 
-        this.storageEngine = (ReadOnlyStorageEngine) AdminServiceRequestHandler.getStorageEngine(storeRepository,
+        this.storageEngine = AdminServiceRequestHandler.getReadOnlyStorageEngine(metadataStore,
-                                                                                                 request.getStore());
+                                                                                 storeRepository,
+                                                                                 request.getStore());
 
         this.blockSize = voldemortConfig.getAllProps()
                                         .getLong("partition.buffer.size.bytes",