rhbz988202 - rate limit REST api plus test

functional-test/src/main/java/org/zanata/page/administration/ServerConfigurationPage.java

@@ -0,0 +1,41 @@
+package org.zanata.page.administration;
+
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.support.FindBy;
+import org.zanata.page.BasePage;
+import org.zanata.util.WebElementUtil;
+
+/**
+ * @author Patrick Huang
+ *         <a href="mailto:pahuang@redhat.com">pahuang@redhat.com</a>
+ */
+public class ServerConfigurationPage extends BasePage {
+    @FindBy(id = "serverConfigForm:urlField")
+    private WebElement urlField;
+
+    @FindBy(id = "serverConfigForm:rateLimitField:rateLimitEml")
+    private WebElement rateLimitField;
+
+    @FindBy(id = "serverConfigForm:save")
+    private WebElement saveButton;
+
+    public ServerConfigurationPage(WebDriver driver) {
+        super(driver);
+    }
+
+    public ServerConfigurationPage inputRateLimit(int limit) {
+        rateLimitField.clear();
+        rateLimitField.sendKeys(limit + "");
+        return this;
+    }
+
+    public String getRateLimit() {
+        return rateLimitField.getAttribute("value");
+    }
+
+    public AdministrationPage save() {
+        saveButton.click();
+        return new AdministrationPage(getDriver());
+    }
+}

functional-test/src/main/java/org/zanata/util/SampleProjectProfile.java

@@ -1,5 +1,6 @@
 package org.zanata.util;
 
+import java.util.List;
 import javax.persistence.EntityManager;
 import javax.persistence.EntityManagerFactory;
 import javax.persistence.Query;
@@ -45,6 +46,8 @@
 import com.github.huangp.entityunit.entity.EntityMakerBuilder;
 import com.github.huangp.entityunit.entity.FixIdCallback;
 import com.github.huangp.entityunit.maker.FixedValueMaker;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import lombok.extern.slf4j.Slf4j;
 
@@ -92,12 +95,29 @@ public void deleteExceptEssentialData() {
         enUSLocale =
                 forLocale(false, LocaleId.EN_US).makeAndPersist(entityManager,
                         HLocale.class);
-        Query query = entityManager.createQuery(
-                "update HApplicationConfiguration set value = '' where key = :key");
-        query.setParameter("key", HApplicationConfiguration.KEY_HOME_CONTENT);
-        query.executeUpdate();
-        query.setParameter("key", HApplicationConfiguration.KEY_HELP_CONTENT);
-        query.executeUpdate();
+
+        List<HApplicationConfiguration> configurations = entityManager
+                .createQuery("from HApplicationConfiguration",
+                        HApplicationConfiguration.class).getResultList();
+
+        Iterable<HApplicationConfiguration> cleanableConfig =
+                Iterables.filter(configurations,
+                        new Predicate<HApplicationConfiguration>() {
+                            @Override
+                            public boolean
+                                    apply(HApplicationConfiguration input) {
+                                String key = input.getKey();
+                                // TODO if we ever need to test this settings, we need to reset values for following to default
+                                return !key
+                                        .equals(HApplicationConfiguration.KEY_EMAIL_FROM_ADDRESS)
+                                        && !key.equals(HApplicationConfiguration.KEY_EMAIL_LOG_EVENTS)
+                                        && !key.equals(HApplicationConfiguration.KEY_EMAIL_LOG_LEVEL);
+                            }
+                        });
+        for (HApplicationConfiguration configuration : cleanableConfig) {
+            entityManager.remove(configuration);
+        }
+        entityManager.flush();
 
         purgeLuceneIndexes();
         // TODO probably should delete cache as well

functional-test/src/test/java/org/zanata/feature/misc/RateLimitTest.java

@@ -0,0 +1,157 @@
+package org.zanata.feature.misc;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.hamcrest.Matchers;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.zanata.feature.DetailedTest;
+import org.zanata.page.administration.AdministrationPage;
+import org.zanata.page.administration.ServerConfigurationPage;
+import org.zanata.util.AddUsersRule;
+import org.zanata.util.ZanataRestCaller;
+import org.zanata.workflow.BasicWorkFlow;
+import org.zanata.workflow.LoginWorkFlow;
+import com.google.common.base.Function;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+
+/**
+ * @author Patrick Huang <a
+ *         href="mailto:pahuang@redhat.com">pahuang@redhat.com</a>
+ */
+@Category(DetailedTest.class)
+@Slf4j
+public class RateLimitTest {
+    @ClassRule
+    public static AddUsersRule addUsersRule = new AddUsersRule();
+
+    private static final String TRANSLATOR = "translator";
+    private static final String TRANSLATOR_API =
+            "d83882201764f7d339e97c4b087f0806";
+
+    @Test
+    public void canConfigureRateLimit() {
+        new LoginWorkFlow().signIn("admin", "admin");
+        BasicWorkFlow basicWorkFlow = new BasicWorkFlow();
+        ServerConfigurationPage serverConfigPage =
+                basicWorkFlow.goToPage("admin/server_configuration",
+                        ServerConfigurationPage.class);
+
+        assertThat(serverConfigPage.getRateLimit(), Matchers.isEmptyString());
+
+        AdministrationPage administrationPage =
+                serverConfigPage.inputRateLimit(1).save();
+
+        assertThat(administrationPage.getNotificationMessage(),
+                Matchers.equalTo("Configuration was successfully updated."));
+
+        serverConfigPage =
+                basicWorkFlow.goToPage("admin/server_configuration",
+                        ServerConfigurationPage.class);
+        assertThat(serverConfigPage.getRateLimit(), Matchers.equalTo("1"));
+    }
+
+    @Test
+    public void canRateLimitRestRequestsPerAPIKey() throws InterruptedException {
+        new LoginWorkFlow().signIn("admin", "admin");
+        BasicWorkFlow basicWorkFlow = new BasicWorkFlow();
+        ServerConfigurationPage serverConfigPage =
+                basicWorkFlow.goToPage("admin/server_configuration",
+                        ServerConfigurationPage.class);
+        // adjust rate limit per second
+        serverConfigPage.inputRateLimit(3).save();
+
+        // prepare to fire multiple REST requests
+        final AtomicInteger atomicInteger = new AtomicInteger(1);
+        // translator creates the project/version
+        final String projectSlug = "project";
+        final String iterationSlug = "version";
+        new ZanataRestCaller(TRANSLATOR, TRANSLATOR_API)
+                .createProjectAndVersion(projectSlug, iterationSlug, "gettext");
+
+        // requests from translator user
+        final int translatorThreads = 4;
+        Callable<Integer> translatorTask = new Callable<Integer>() {
+
+            @Override
+            public Integer call() {
+                return invokeRestService(new ZanataRestCaller(TRANSLATOR,
+                        TRANSLATOR_API), projectSlug, iterationSlug,
+                        atomicInteger);
+            }
+        };
+        List<Callable<Integer>> translatorTasks =
+                Collections.nCopies(translatorThreads, translatorTask);
+
+        // requests from admin user
+        int adminThreads = 3;
+        Callable<Integer> adminTask = new Callable<Integer>() {
+            @Override
+            public Integer call() throws Exception {
+                return invokeRestService(new ZanataRestCaller(), projectSlug,
+                        iterationSlug, atomicInteger);
+            }
+        };
+
+        List<Callable<Integer>> adminTasks =
+                Collections.nCopies(adminThreads, adminTask);
+
+        ExecutorService executorService =
+                Executors.newFixedThreadPool(translatorThreads + adminThreads);
+
+        List<Callable<Integer>> tasks =
+                ImmutableList.<Callable<Integer>> builder()
+                        .addAll(translatorTasks).addAll(adminTasks).build();
+
+        List<Future<Integer>> futures = executorService.invokeAll(tasks);
+
+        List<Integer> result = getResultStatusCodes(futures);
+
+        // 1 request from translator should get 503 and fail
+        log.info("result: {}", result);
+        assertThat(result, Matchers.containsInAnyOrder(201, 201, 201, 201, 201,
+                201, 403));
+    }
+
+    private static Integer invokeRestService(ZanataRestCaller restCaller,
+            String projectSlug, String iterationSlug,
+            AtomicInteger atomicInteger) {
+        try {
+            int counter = atomicInteger.getAndIncrement();
+            return restCaller.postSourceDocResource(projectSlug, iterationSlug,
+                    ZanataRestCaller.buildSourceResource("doc" + counter,
+                            ZanataRestCaller.buildTextFlow("res" + counter,
+                                    "content" + counter)), false);
+        } catch (Exception e) {
+            log.info("rest call failed: {}", e.getMessage());
+            return 500;
+        }
+    }
+
+    private static List<Integer> getResultStatusCodes(List<Future<Integer>> futures) {
+        return Lists.transform(futures,
+                new Function<Future<Integer>, Integer>() {
+                    @Override
+                    public Integer apply(Future<Integer> input) {
+                        try {
+                            return input.get();
+                        }
+                        catch (Exception e) {
+                            throw Throwables.propagate(e);
+                        }
+                    }
+                });
+    }
+}

zanata-war/pom.xml

@@ -1205,6 +1205,12 @@
       <artifactId>commons-collections</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.isomorphism</groupId>
+      <artifactId>token-bucket</artifactId>
+      <version>1.1</version>
+    </dependency>
+
     <!-- Seam Dependencies -->
 
     <dependency>

zanata-war/src/main/java/org/zanata/ApplicationConfiguration.java

@@ -391,4 +391,14 @@ private String getBaseWebAssetsUrl() {
         return Objects.firstNonNull(jndiBackedConfig.getWebAssetsUrlBase(),
                 "//assets-zanata.rhcloud.com");
     }
+
+    public int getRateLimitPerSecond() {
+        String limitPerSecond =
+                databaseBackedConfig.getRateLimitPerSecond();
+        if (Strings.isNullOrEmpty(limitPerSecond)) {
+            // default no limit
+            return 0;
+        }
+        return Integer.parseInt(limitPerSecond);
+    }
 }

zanata-war/src/main/java/org/zanata/action/ServerConfigurationBean.java

@@ -96,6 +96,7 @@ public class ServerConfigurationBean implements Serializable {
     private String termsOfUseUrl;
 
     @Digits(integer = 10, fraction = 0)
+    // TODO this won't allow empty string
     private String rateLimitPerSecond;
 
     public String getHomeContent() {

zanata-war/src/main/java/org/zanata/annotation/RateLimiting.java

@@ -0,0 +1,11 @@
+package org.zanata.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.METHOD)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface RateLimiting {
+}

zanata-war/src/main/java/org/zanata/annotation/RateLimitingResource.java

@@ -0,0 +1,15 @@
+package org.zanata.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.jboss.seam.annotations.intercept.Interceptors;
+import org.zanata.seam.interceptor.RateLimitingInterceptor;
+
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+@Interceptors(RateLimitingInterceptor.class)
+public @interface RateLimitingResource {
+}

zanata-war/src/main/java/org/zanata/rest/service/AsynchronousProcessResourceService.java

@@ -29,6 +29,7 @@
 import org.jboss.seam.annotations.In;
 import org.jboss.seam.annotations.Name;
 import org.jboss.seam.annotations.Transactional;
+import org.zanata.annotation.RateLimitingResource;
 import org.zanata.async.AsyncTaskHandle;
 import org.zanata.async.SimpleAsyncTask;
 import org.zanata.common.EntityStatus;
@@ -70,6 +71,7 @@
 @Name("asynchronousProcessResourceService")
 @Path(AsynchronousProcessResource.SERVICE_PATH)
 @Transactional
+@RateLimitingResource
 @Slf4j
 public class AsynchronousProcessResourceService implements
         AsynchronousProcessResource {

zanata-war/src/main/java/org/zanata/rest/service/SourceDocResourceService.java

@@ -45,6 +45,8 @@
 import org.jboss.seam.annotations.Name;
 import org.jboss.seam.annotations.Transactional;
 import org.jboss.seam.annotations.security.Restrict;
+import org.zanata.annotation.RateLimiting;
+import org.zanata.annotation.RateLimitingResource;
 import org.zanata.common.EntityStatus;
 import org.zanata.common.LocaleId;
 import org.zanata.dao.DocumentDAO;
@@ -71,6 +73,7 @@
 @Path(SourceDocResource.SERVICE_PATH)
 @Slf4j
 @Transactional
+@RateLimitingResource
 public class SourceDocResourceService implements SourceDocResource {
 
     @Context
@@ -148,6 +151,7 @@ public Response get(Set<String> extensions) {
     }
 
     @Override
+    @RateLimiting
     @Restrict("#{s:hasPermission(sourceDocResourceService.securedIteration, 'import-template')}")
     public
             Response post(Resource resource, Set<String> extensions,