Permalink
Browse files

Fixed read-only store grandfathering script to now output metadata

  • Loading branch information...
1 parent 31387a8 commit 64028c529bf8213f9f6906264eda8976e09c5218 @rsumbaly rsumbaly committed Oct 6, 2010
@@ -15,39 +15,51 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+# One time grandfathering script to convert all read-only store dirs to
+# new format
if [ $# -ne 1 ];
then
echo 'USAGE: bin/grandfather-readonly.sh [readonly-folder]'
exit 1
fi
-# Read args
+# Read args
READ_ONLY_DIR=$1
+# Create temporary metadata file
+METADATA_FILE="/tmp/$(basename $0).$$.tmp"
+echo "{\"format\":\"ro0\"}" > $METADATA_FILE
+
for stores in $READ_ONLY_DIR/*
do
if [ -d $stores ]; then
+ echo ---Working on store ${stores} ---
# Convert all to .temp
numVersions=`find $stores -name version-* | grep -v .bak | grep -v .temp | wc -l`
+ maxVersion=`find $stores -name version-* | grep -v .bak | grep -v .temp | awk -F'-' '{print $2}' | sort -n | tail -1`
+ if [ $numVersions -eq 1 ]; then
+ cp $METADATA_FILE ${stores}/version-${maxVersion}/.metadata
+ echo Added metadata to ${stores}/version-${maxVersion}
+ fi
if [ $numVersions -gt 1 ]; then
- maxVersion=`find $stores -name version-* | grep -v .bak | grep -v .temp | awk -F'-' '{print $2}' | sort -n | tail -1`
for versionDirNo in `find $stores -name version-* | grep -v .bak | grep -v .temp | awk -F'-' '{print $2}' | sort -n`
do
- echo $stores/version-$versionDirNo $stores/version-$maxVersion.temp
- mv $stores/version-$versionDirNo $stores/version-$maxVersion.temp
+ mv ${stores}/version-${versionDirNo} ${stores}/version-${maxVersion}.temp
+ echo Moved ${stores}/version-${versionDirNo} to ${stores}/version-${maxVersion}.temp
+ cp $METADATA_FILE ${stores}/version-${maxVersion}.temp/.metadata
+ echo Added metadata to ${stores}/version-${maxVersion}.temp
let maxVersion=maxVersion-1
done
- echo '-------'
fi
# Convert all .temp to normal
numVersionsTmp=`find $stores -name version-*.temp | grep -v .bak | wc -l`
if [ $numVersionsTmp -gt 1 ]; then
for versionDir in `find $stores -name version-*.temp | grep -v .bak | awk -F'.' '{print $1}'`
do
- mv $versionDir.temp $versionDir
+ mv ${versionDir}.temp ${versionDir}
+ echo Moved ${versionDir}.temp to ${versionDir}
done
- echo '======='
fi
fi
done
@@ -132,6 +132,7 @@ private void setFetcherClass(VoldemortServer server) {
@Override
public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException,
IOException {
+ initStores((VoldemortServer) getServletContext().getAttribute(VoldemortServletContextListener.SERVER_KEY));
Map<String, Object> params = Maps.newHashMap();
params.put("stores", stores);
velocityEngine.render("read-only-mgmt.vm", params, resp.getOutputStream());
@@ -162,7 +163,6 @@ private void doSwap(HttpServletRequest req, HttpServletResponse resp) throws IOE
ServletException {
String dir = getRequired(req, "dir");
String storeName = getRequired(req, "store");
- String formatString = getOptional(req, "format");
ReadOnlyStorageEngine store = this.getStore(storeName);
if(store == null)
@@ -200,15 +200,21 @@ private void doFetch(HttpServletRequest req, HttpServletResponse resp) throws IO
File fetchDir = null;
if(fileFetcher == null) {
- logger.warn("File fetcher class has not instantiated correctly");
- fetchDir = new File(fetchUrl);
+ logger.warn("File fetcher class has not instantiated correctly. Assuming local file");
+
+ if(!Utils.isReadableDir(fetchUrl)) {
+ throw new VoldemortException("Fetch url " + fetchUrl + " is not readable");
+ }
+
+ fetchDir = new File(store.getStoreDirPath(), "version-" + Long.toString(pushVersion));
+ Utils.move(new File(fetchUrl), fetchDir);
} else {
logger.info("Executing fetch of " + fetchUrl);
try {
fetchDir = fileFetcher.fetch(fetchUrl, store.getStoreDirPath() + File.separator
- + "version-" + pushVersion);
+ + "version-" + Long.toString(pushVersion));
if(fetchDir == null) {
throw new ServletException("File fetcher failed for " + fetchUrl
@@ -255,6 +261,7 @@ private String getRequired(HttpServletRequest req, String name) throws ServletEx
}
private ReadOnlyStorageEngine getStore(String storeName) throws ServletException {
+ initStores((VoldemortServer) getServletContext().getAttribute(VoldemortServletContextListener.SERVER_KEY));
for(ReadOnlyStorageEngine store: this.stores)
if(store.getName().equals(storeName))
return store;
@@ -237,8 +237,9 @@ public StreamRequestHandler handleRequest(final DataInputStream inputStream,
try {
for(String storeName: storeNames) {
- ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
- storeName);
+ ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
+ storeRepository,
+ storeName);
VAdminProto.ROStoreVersionDirMap storeResponse = VAdminProto.ROStoreVersionDirMap.newBuilder()
.setStoreName(storeName)
.setStoreDir(store.getCurrentDirPath())
@@ -260,8 +261,9 @@ public StreamRequestHandler handleRequest(final DataInputStream inputStream,
try {
for(String storeName: storeNames) {
- ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
- storeName);
+ ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
+ storeRepository,
+ storeName);
File storeDirPath = new File(store.getStoreDirPath());
if(!storeDirPath.exists())
@@ -406,8 +408,9 @@ public StreamRequestHandler handleUpdatePartitionEntries(VAdminProto.UpdateParti
VAdminProto.RollbackStoreResponse.Builder response = VAdminProto.RollbackStoreResponse.newBuilder();
try {
- ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
- storeName);
+ ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
+ storeRepository,
+ storeName);
File rollbackVersionDir = new File(store.getStoreDirPath(), "version-" + pushVersion);
@@ -446,8 +449,10 @@ public StreamRequestHandler handleUpdatePartitionEntries(VAdminProto.UpdateParti
}
private void swapStore(String storeName, String directory) throws VoldemortException {
- ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
- storeName);
+
+ ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
+ storeRepository,
+ storeName);
if(!Utils.isReadableDir(directory))
throw new VoldemortException("Store directory '" + directory
@@ -488,8 +493,9 @@ private void swapStore(String storeName, String directory) throws VoldemortExcep
.setDescription("Fetch store")
.setStatus("started");
try {
- final ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
- storeName);
+ final ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
+ storeRepository,
+ storeName);
final long pushVersion;
if(request.hasPushVersion()) {
pushVersion = request.getPushVersion();
@@ -979,6 +985,15 @@ static int valueSize(Versioned<byte[]> value) {
return value.getValue().length + ((VectorClock) value.getVersion()).sizeInBytes() + 1;
}
+ static ReadOnlyStorageEngine getReadOnlyStorageEngine(MetadataStore metadata,
+ StoreRepository repo,
+ String name) {
+ if(metadata.getStoreDef(name).getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) != 0)
+ throw new VoldemortException("Store " + name
+ + " is not a read-only store, cannot complete operation");
+ return (ReadOnlyStorageEngine) getStorageEngine(repo, name);
+ }
+
static StorageEngine<ByteArray, byte[], byte[]> getStorageEngine(StoreRepository storeRepository,
String storeName) {
StorageEngine<ByteArray, byte[], byte[]> storageEngine = storeRepository.getStorageEngine(storeName);
@@ -49,8 +49,9 @@ protected FetchPartitionFileStreamRequestHandler(VAdminProto.FetchPartitionFiles
throw new VoldemortException("Should be fetching partition files only for read-only stores");
}
- this.storageEngine = (ReadOnlyStorageEngine) AdminServiceRequestHandler.getStorageEngine(storeRepository,
- request.getStore());
+ this.storageEngine = AdminServiceRequestHandler.getReadOnlyStorageEngine(metadataStore,
+ storeRepository,
+ request.getStore());
this.blockSize = voldemortConfig.getAllProps()
.getLong("partition.buffer.size.bytes",

0 comments on commit 64028c5

Please sign in to comment.