Permalink
Browse files

Add page for swapping data files for read only stores.

  • Loading branch information...
jkreps committed Jan 20, 2009
1 parent 492b029 commit 9bb13d70f015b5086f662ba1dc708c2bcab4d896
View
12 BUGS
@@ -1,12 +0,0 @@
-1. Connection churn on mac os with many threads, not on Linux.
-2. Sporadic exception in read repair under very, very high concurrency:
- Exception in thread "pool-2-thread-2" java.util.NoSuchElementException
- at java.util.HashMap$HashIterator.nextEntry(HashMap.java:844)
- at java.util.HashMap$KeyIterator.next(HashMap.java:877)
- at com.google.common.collect.StandardMultimap$WrappedCollection$WrappedIterator.next(StandardMultimap.java:469)
- at voldemort.store.routed.ReadRepairer.getRepairs(ReadRepairer.java:86)
- at voldemort.store.routed.RoutedStore$3.run(RoutedStore.java:293)
- at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:650)
- at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:675)
- at java.lang.Thread.run(Thread.java:613)
-3. Deletes should not attempt on unavailable nodes
View
6 NOTES
@@ -1,5 +1,7 @@
Getting Started
+For the most up-to-date information see http://project-voldemort.com
+
## checkout and build
jkreps@jkreps-md:/tmp > svn co svn+ssh://cm01.corp/lirepo/voldemort/trunk voldemort
jkreps@jkreps-md:/tmp > cd voldemort
@@ -53,4 +55,6 @@ Background Resources
- OpenDHT and Bamboo papers
- BDB Performance: http://www.oracle.com/technology/products/berkeley-db/pdf/berkeley-db-perf.pdf
- Origin of vector clocks: http://research.microsoft.com/users/lamport/pubs/time-clocks.pdf
-- Brewer's conjecture: http://citeseer.ist.psu.edu/544596.html
+- Brewer's conjecture: http://citeseer.ist.psu.edu/544596.html
+
+Current build is from r19
View
@@ -1,6 +1,6 @@
#Build Number for ANT. Do not edit!
-#Sun Jan 18 18:24:31 PST 2009
-build.number=167
+#Mon Jan 19 18:23:15 PST 2009
+build.number=173
>>>>>>>=1.8
=\=\=\=\=\=\=
<<<<<<<=build_number.txt
@@ -186,6 +186,13 @@ public Cluster getCluster() {
return services;
}
+ public VoldemortService getService(String name) {
+ for(VoldemortService service: services)
+ if(service.getName().equals(name))
+ return service;
+ return null;
+ }
+
public ConcurrentMap<String, Store<byte[], byte[]>> getStoreMap() {
return storeMap;
}
@@ -29,6 +29,7 @@
import voldemort.server.AbstractService;
import voldemort.server.VoldemortServer;
import voldemort.server.http.gui.AdminServlet;
+import voldemort.server.http.gui.ReadOnlyStoreManagementServlet;
import voldemort.server.http.gui.VelocityEngine;
/**
@@ -71,9 +72,14 @@ public void startInner() {
httpServer.setSendDateHeader(false);
Context context = new Context(httpServer, "/", Context.NO_SESSIONS);
context.setAttribute(VoldemortServletContextListener.SERVER_CONFIG_KEY, server);
+ context.setAttribute(VoldemortServletContextListener.VELOCITY_ENGINE_KEY,
+ velocityEngine);
context.addServlet(new ServletHolder(new AdminServlet(server, velocityEngine)),
"/admin");
context.addServlet(new ServletHolder(new StoreServlet(server.getStoreMap())), "/*");
+ context.addServlet(new ServletHolder(new ReadOnlyStoreManagementServlet(server,
+ velocityEngine)),
+ "/read-only/mgmt");
this.context = context;
this.httpServer = httpServer;
this.httpServer.start();
@@ -55,6 +55,7 @@ public AdminServlet(VoldemortServer server, VelocityEngine engine) {
public void init() throws ServletException {
super.init();
this.server = (VoldemortServer) Utils.notNull(getServletContext().getAttribute(VoldemortServletContextListener.SERVER_CONFIG_KEY));
+ this.velocityEngine = (VelocityEngine) Utils.notNull(getServletContext().getAttribute(VoldemortServletContextListener.VELOCITY_ENGINE_KEY));
}
@Override
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2008-2009 LinkedIn, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.server.http.gui;
+
+import java.io.IOException;
+import java.util.Map;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import voldemort.server.VoldemortServer;
+import voldemort.server.http.VoldemortServletContextListener;
+import voldemort.server.storage.StorageService;
+import voldemort.store.readonly.RandomAccessFileStore;
+import voldemort.utils.Utils;
+
+import com.google.common.collect.Maps;
+
+public class ReadOnlyStoreManagementServlet extends HttpServlet {
+
+ private static final long serialVersionUID = 1;
+
+ private Map<String, RandomAccessFileStore> stores;
+ private VelocityEngine velocityEngine;
+
+ public ReadOnlyStoreManagementServlet(VoldemortServer server, VelocityEngine engine) {
+ this.stores = getReadOnlyStores(server);
+ this.velocityEngine = Utils.notNull(engine);
+ }
+
+ @Override
+ public void init() throws ServletException {
+ super.init();
+ this.stores = getReadOnlyStores((VoldemortServer) getServletContext().getAttribute(VoldemortServletContextListener.SERVER_CONFIG_KEY));
+ this.velocityEngine = (VelocityEngine) Utils.notNull(getServletContext().getAttribute(VoldemortServletContextListener.VELOCITY_ENGINE_KEY));
+ }
+
+ private Map<String, RandomAccessFileStore> getReadOnlyStores(VoldemortServer server) {
+ StorageService storage = (StorageService) Utils.notNull(server)
+ .getService("storage-service");
+ return storage.getReadOnlyStores();
+ }
+
+ @Override
+ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException,
+ IOException {
+ Map<String, Object> params = Maps.newHashMap();
+ params.put("stores", stores);
+ velocityEngine.render("read-only-mgmt.vm", params, resp.getOutputStream());
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ if("swap".equals(getRequired(req, "operation"))) {
+ String indexFile = getRequired(req, "index");
+ String dataFile = getRequired(req, "data");
+ String storeName = getRequired(req, "store");
+ if(!stores.containsKey(storeName))
+ throw new ServletException("'" + storeName
+ + "' is not a registered read-only store.");
+ if(!Utils.isReadableFile(indexFile))
+ throw new ServletException("Index file '" + indexFile + "' is not a readable file.");
+ if(!Utils.isReadableFile(dataFile))
+ throw new ServletException("Data file '" + dataFile + "' is not a readable file.");
+
+ RandomAccessFileStore store = stores.get(storeName);
+ store.swapFiles(indexFile, dataFile);
+ resp.getWriter().write("Swap completed.");
+ } else {
+ throw new IllegalArgumentException("Unknown operation parameter: "
+ + req.getParameter("operation"));
+ }
+ }
+
+ private String getRequired(HttpServletRequest req, String name) throws ServletException {
+ String val = req.getParameter(name);
+ if(name == null)
+ throw new ServletException("Missing required paramter '" + name + "'.");
+ return val;
+ }
+}
@@ -0,0 +1,70 @@
+##
+## Copyright 2008-2009 LinkedIn, Inc
+##
+## Licensed under the Apache License, Version 2.0 (the "License"); you may not
+## use this file except in compliance with the License. You may obtain a copy of
+## the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+## License for the specific language governing permissions and limitations under
+## the License.
+##
+
+<html>
+ <head>
+ <title>Voldemort Administration</title>
+ <style>
+ h1 {font-weight: bold;
+ font-size: 30pt;
+ text-align: center}
+ h2 {font-weight: bold;
+ font-size: 20pt;
+ text-align: center}
+ body {background-color: darkred}
+ .content {background-color: white;
+ border: 3px solid black;
+ margin: 20px;
+ margin-left: 100px;
+ margin-right: 100px;
+ padding: 10px;
+ min-height: 800px}
+ </style>
+ </head>
+ <body>
+ <div class="content">
+ <h2>Swap Data Files</h2>
+ <form method="post">
+ <input type="hidden" name="operation" value="swap"/>
+ <table align="center">
+ <tr>
+ <td>Store</td>
+ <td>
+ <select name="store">
+ #foreach($store in $stores.keySet())
+ <option value="$store">$store</option>
+ #end
+ </select>
+ </td>
+ </tr>
+ <tr>
+ <td>Index File</td>
+ <td><input type="text" name="index"/></td>
+ </tr>
+ <tr>
+ <td>Data File</td>
+ <td><input type="text" name="data"/></td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <input type="submit" value="Submit"/>
+ </td>
+ </tr>
+ </table>
+ </form>
+ </div>
+ </body>
+</html>
@@ -57,6 +57,7 @@
import voldemort.store.metadata.MetadataStore;
import voldemort.store.mysql.MysqlStorageConfiguration;
import voldemort.store.readonly.RandomAccessFileStorageConfiguration;
+import voldemort.store.readonly.RandomAccessFileStore;
import voldemort.store.serialized.SerializingStore;
import voldemort.store.slop.Slop;
import voldemort.store.slop.SlopDetectingStore;
@@ -83,6 +84,7 @@
private final ConcurrentMap<StorageEngineType, StorageConfiguration> storageConfigurations;
private final StoreDefinitionsMapper storeMapper;
private final SchedulerService scheduler;
+ private final Map<String, RandomAccessFileStore> readOnlyStores;
private MetadataStore metadataStore;
private Store<byte[], Slop> slopStore;
@@ -100,6 +102,7 @@ public StorageService(String name,
this.metadataStore = new MetadataStore(new FilesystemStorageEngine(MetadataStore.METADATA_STORE_NAME,
config.getMetadataDirectory()),
storeMap);
+ this.readOnlyStores = new ConcurrentHashMap<String, RandomAccessFileStore>();
}
private ConcurrentMap<StorageEngineType, StorageConfiguration> initStorageConfigurations(VoldemortConfig config) {
@@ -139,6 +142,9 @@ protected void startInner() {
StorageEngine<byte[], byte[]> engine = getStore(def.getName(), def.getType());
rawEngines.put(engine.getName(), engine);
+ if(def.getType().equals(StorageEngineType.READONLY))
+ this.readOnlyStores.put(engine.getName(), (RandomAccessFileStore) engine);
+
/* Now add any store wrappers that are enabled */
Store<byte[], byte[]> store = engine;
if(voldemortConfig.isSlopDetectionEnabled()) {
@@ -265,6 +271,10 @@ public MetadataStore getMetadataStore() {
return this.metadataStore;
}
+ public Map<String, RandomAccessFileStore> getReadOnlyStores() {
+ return this.readOnlyStores;
+ }
+
public Store<byte[], Slop> getSlopStore() {
return this.slopStore;
}
@@ -161,8 +161,10 @@ public ExternalSorter(Serializer<V> serializer,
logger.info("Chunk " + chunkId + ": filling sort buffer for chunk...");
@SuppressWarnings("unchecked")
final V[] buffer = (V[]) new Object[internalSortSize];
- for(int i = 0; i < internalSortSize && input.hasNext(); i++)
- buffer[i] = input.next();
+ int chunkSizeIter = 0;
+ for(; chunkSizeIter < internalSortSize && input.hasNext(); chunkSizeIter++)
+ buffer[chunkSizeIter] = input.next();
+ final int chunkSize = chunkSizeIter;
logger.info("Chunk " + chunkId + ": sort buffer filled...adding to sort queue.");
// sort and write out asynchronously
@@ -171,7 +173,7 @@ public ExternalSorter(Serializer<V> serializer,
public void run() {
logger.info("Chunk " + chunkId + ": sorting buffer.");
long start = System.currentTimeMillis();
- Arrays.sort(buffer, comparator);
+ Arrays.sort(buffer, 0, chunkSize, comparator);
long ellapsed = System.currentTimeMillis() - start;
logger.info("Chunk " + chunkId + ": sort completed in " + ellapsed
+ " ms, writing to temp file.");
@@ -183,8 +185,8 @@ public void run() {
tempFiles.add(tempFile);
DataOutputStream output = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tempFile),
bufferSize));
- for(V v: buffer)
- writeValue(output, v);
+ for(int i = 0; i < chunkSize; i++)
+ writeValue(output, buffer[i]);
output.close();
} catch(IOException e) {
throw new VoldemortException(e);
@@ -17,34 +17,56 @@
package voldemort.store.readonly;
import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
import voldemort.server.VoldemortConfig;
import voldemort.store.StorageConfiguration;
import voldemort.store.StorageEngine;
import voldemort.store.StorageEngineType;
+import voldemort.utils.JmxUtils;
public class RandomAccessFileStorageConfiguration implements StorageConfiguration {
private final int numFileHandles;
private final int numBackups;
private final long fileAccessWaitTimeoutMs;
private final File storageDir;
+ private final Set<ObjectName> registeredBeans;
public RandomAccessFileStorageConfiguration(VoldemortConfig config) {
this.numFileHandles = config.getReadOnlyStorageFileHandles();
this.storageDir = new File(config.getReadOnlyDataStorageDirectory());
this.fileAccessWaitTimeoutMs = config.getReadOnlyFileWaitTimeoutMs();
this.numBackups = config.getReadOnlyBackups();
+ this.registeredBeans = Collections.synchronizedSet(new HashSet<ObjectName>());
}
- public void close() {}
+ public void close() {
+ MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+ for(ObjectName name: registeredBeans)
+ JmxUtils.unregisterMbean(server, name);
+ }
public StorageEngine<byte[], byte[]> getStore(String name) {
- return new RandomAccessFileStore(name,
- storageDir,
- numBackups,
- numFileHandles,
- fileAccessWaitTimeoutMs);
+ RandomAccessFileStore store = new RandomAccessFileStore(name,
+ storageDir,
+ numBackups,
+ numFileHandles,
+ fileAccessWaitTimeoutMs);
+ ObjectName objName = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()),
+ name);
+ JmxUtils.registerMbean(ManagementFactory.getPlatformMBeanServer(),
+ JmxUtils.createModelMBean(store),
+ objName);
+ registeredBeans.add(objName);
+
+ return store;
}
public StorageEngineType getType() {
Oops, something went wrong.

0 comments on commit 9bb13d7

Please sign in to comment.