From 3bc3d81e964aac59f61951740e848bd429a15b3c Mon Sep 17 00:00:00 2001 From: Furkan KAMACI Date: Tue, 23 Aug 2016 23:41:43 +0300 Subject: [PATCH] NUTCH-2301 Tests for Security Layer of NutchServer Are Created --- src/test/nutch-site.xml | 48 ++++ src/test/nutch-ssl.keystore.jks | Bin 0 -> 2300 bytes .../nutch/api/AbstractNutchAPITestBase.java | 186 +++++++++++++++ src/test/org/apache/nutch/api/TestAPI.java | 225 ------------------ .../org/apache/nutch/api/TestNutchAPI.java | 100 ++++++++ 5 files changed, 334 insertions(+), 225 deletions(-) create mode 100644 src/test/nutch-ssl.keystore.jks create mode 100644 src/test/org/apache/nutch/api/AbstractNutchAPITestBase.java delete mode 100644 src/test/org/apache/nutch/api/TestAPI.java create mode 100644 src/test/org/apache/nutch/api/TestNutchAPI.java diff --git a/src/test/nutch-site.xml b/src/test/nutch-site.xml index e599547bab..4dbce0cb53 100644 --- a/src/test/nutch-site.xml +++ b/src/test/nutch-site.xml @@ -29,4 +29,52 @@ + + restapi.auth + NONE + + Configures authentication type for communicating with RESTAPI. Valid values are BASIC, DIGEST, SSL and NONE. + When no authentication type is defined NONE will be used as default which does not provide security. + Use the restapi.auth.username and restapi.auth.password properties to configure + your credentials if security is used. + + + + + restapi.auth.users + admin|admin|admin,user|user|user + + Username, password and role combination for REST API authentication/authorization. restapi.auth property should be set to either BASIC or DIGEST to use this property. + Username, password and role should be delimited by pipe character (|) Every user should be separated with comma character (,). i.e. admin|admin|admin,user|user|user. + Default is admin|admin|admin,user|user|user + + + + + restapi.auth.ssl.storepath + nutch-ssl.keystore.jks + + Key store path for jks file. restapi.auth property should be set to SSL to use this property. + etc/nutch-ssl.keystore.jks is used for restapi.auth.ssl.storepath as default. + + + + + restapi.auth.ssl.storepass + password + + Key store path for jks file. restapi.auth property should be set to SSL to use this property. + "password" is used for restapi.auth.ssl.storepass as default. + + + + + restapi.auth.ssl.keypass + password + + Key store path for jks file. restapi.auth property should be set to SSL to use this property. + "password" is used for restapi.auth.ssl.keypass as default. + + + diff --git a/src/test/nutch-ssl.keystore.jks b/src/test/nutch-ssl.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..9d0bd01714e4031b3ce5e02acff54db83073eac1 GIT binary patch literal 2300 zcmd5-`9Bkm8{cM*xtSx%F>;G-v=}*ZWOIxhl_Te(uwj!sOzx{va^wg(D$?dYYl$L7 za;3SMd_vKZL|^s&{_yz|zCS!ayx*_qdA*+3`~AG$&;Hu}8UO$Q9Tf0yu@gec-hKzj zspS)10{~zL!YSZAI6nus8V3*zR0RnEf$RV<1^lV31RrPLGDO-=UW~ui1*hF9zpras_+(){q4P% z@(`lww;Z9^;%q|)YXs1t%EqqwLsF;=+>p)_Iq(D(Q>R@X}Y!G-S_e6o1~`jBr6eP z*~Sg?+TDSWDzPTIIU3&@(^5*abkpYA(GWyb#J&Wje(xjhNR(GQwVWR-ZqNMu)GC00Nq&U`VV7y+G0&?&G5&)wWY1{psbGdrdnUJP zZ^akLWOI1M6|404`5jRD2Jcmfe^S15%@dlC)0YS*)36}l=UMH8$xCzU=1XO{yzS_0AMl%&$o^xWGB(}dPF3+Kq*7}&_&${1v??~72GpBC(V@>Yp#mh_y(J{Q5^=_*kNh`sA1NiN} zLBG6@+0B)2SQTQKZCpvm6@w1ta$8q7W3NGsdQ!sL>gRqbCLqCuPgh#l6>dq%Tp~An zITYVrS+=pJUKzG6!yMqm!O4ABb2OD0#r)C3X~hgU!(FyT28sq0e0G6f&3h6NPylV z+< zwU8y6Mt64=7P{9?NwGX!`HMrhN`e?rNtuN7Sb6K(5ztsUrMkY$;@h`_Yub*h- zGft^{_Hp=0>ZpeuL+j}07rJenh*W{IdLTQs7_Z9ivlZq&xPiqxB&1R|bA*2nRJ18g zH1DD*aTGBk2lvX9X_?fUhOm?AC1cze6X&4vMZIau!-r!V#B8OO!WFFJ_gOy7#CDp1 z^;g`lp-puod_L>yhV5Lok6TYI4J2oVl@_S<7|Lmj&dz$**d+W6 zp6D~5X>X_k#HMlEP$3Eo*gx@|?5H?R##ARpt#o~O_vL?c@tnqSxGn&|)(EGtJ%Lj| z`K4?iAP5BZ@Axnc=i`8=CRrqH^8tZ8>;QOt3W5g?;b3zFb3j23XApu1`FQ}ipq$p8 zA%UJDo(Lhhz#oPa%7r13Jqcc+0SI|G><`5al{60W^!CHc+7mC5!#qRqvgX84f{!Qp z5|MxqfeZg>;(_wug7Accme3GyJVF94{)c6Q3KBznH9Y^0Yy1szxU@JQ0tMGWY9VyB zbhMDpe;@*ZMEno_Kdnju%Kcl&gEzxQ0SW-%6d*T<0t5nX@cZy)ef~*|8MiYs6`Gx= z_kXFa4iRZjH5Tz=Stz`iyFSpc2wFLJO~M&q6miMLInwc(qb5BM&TTsflW}50N>UCd zJ+HqSn*Cd_Pa5~T@!Pf`df}zx>rYAuA0M3ym{CU!;Fs#1JUMjFSt>gA!?hidwo>U}#F7S@x-U_agWdT%ht5tyblp%*xi zIe%p1>35@D#|c!;b-4xHKBk9qn)B`X)mI&9gKH6(k@x1^zOU=zW5sYN24Bf#OAu=~ z3|e+s{%Ax-1{sv5*}*%4vxx1vdeXf!i@z?&^qr=w*Hg5Eu1WgV3NaP?B9jkyf>-T$ zKD#q!J?jKk!Qgit5oP0I8xVZmBNXS~y`KE;4${IYXHDE*Ot4D}2b=o%Oi2bNj7lNY uc9|9w>^V_PS4r*`8fY}dju~V9V3oCSFWWmtqRIoIE_GUpq`)Jh8T=orc;r0* literal 0 HcmV?d00001 diff --git a/src/test/org/apache/nutch/api/AbstractNutchAPITestBase.java b/src/test/org/apache/nutch/api/AbstractNutchAPITestBase.java new file mode 100644 index 0000000000..3cdff23766 --- /dev/null +++ b/src/test/org/apache/nutch/api/AbstractNutchAPITestBase.java @@ -0,0 +1,186 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.nutch.api; + +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.api.impl.RAMConfManager; +import org.apache.nutch.api.security.AuthenticationTypeEnum; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.BeforeClass; +import org.restlet.data.ChallengeRequest; +import org.restlet.data.ChallengeResponse; +import org.restlet.data.ChallengeScheme; +import org.restlet.resource.ClientResource; +import org.restlet.resource.ResourceException; + +import static org.junit.Assert.assertEquals; + +/** + * Base class for Nutc REST API tests. + * Default path is /admin and default port is 8081. + */ +public abstract class AbstractNutchAPITestBase { + protected static Configuration conf; + protected static final Integer DEFAULT_PORT = 8081; + protected static final String DEFAULT_PATH = "/admin"; + public NutchServer nutchServer; + + /** + * Creates Nutch configuration + * + * @throws Exception + */ + @BeforeClass + public static void before() throws Exception { + conf = NutchConfiguration.create(); + } + + /** + * Tests a request for given expected status code for {@link NutchServer} from which port it runs + * + * @param expectedStatusCode expected status code + * @param port port that {@link NutchServer} runs at + */ + public void testRequest(int expectedStatusCode, int port) { + testRequest(expectedStatusCode, port, null, null, DEFAULT_PATH, null); + } + + /** + * Tests a request for given expected status code for {@link NutchServer} from which port it runs + * with given username and password credentials + * + * @param expectedStatusCode expected status code + * @param port port that {@link NutchServer} runs at + * @param username username + * @param password password + */ + public void testRequest(int expectedStatusCode, int port, String username, String password) { + testRequest(expectedStatusCode, port, username, password, DEFAULT_PATH, null); + } + + /** + * Tests a request for given expected status code for {@link NutchServer} from which port it runs + * with given username and password credentials for a {@link ChallengeScheme} + * + * @param expectedStatusCode expected status code + * @param port port that {@link NutchServer} runs at + * @param username username + * @param password password + * @param challengeScheme challenge scheme + */ + public void testRequest(int expectedStatusCode, int port, String username, String password, ChallengeScheme challengeScheme) { + testRequest(expectedStatusCode, port, username, password, DEFAULT_PATH, challengeScheme); + } + + /** + * Tests a request for given expected status code for {@link NutchServer} from which port it runs + * with given username and password credentials for a {@link ChallengeScheme} + * + * @param expectedStatusCode expected status code + * @param port port that {@link NutchServer} runs at + * @param username username + * @param password password + * @param path path + * @param challengeScheme challenge scheme + */ + public void testRequest(int expectedStatusCode, int port, String username, String password, String path, + ChallengeScheme challengeScheme) { + if (port <= 0) { + port = DEFAULT_PORT; + } + + if (!path.startsWith("/")) { + path += "/"; + } + String protocol = AuthenticationTypeEnum.SSL.toString().equals(conf.get("restapi.auth")) ? "https" : "http"; + ClientResource resource = new ClientResource(protocol + "://localhost:" + port + path); + if (challengeScheme != null) { + resource.setChallengeResponse(challengeScheme, username, password); + } + + try { + resource.get(); + } catch (ResourceException rex) { + //Don't use it. Use the status code to check afterwards + } + + /* + If request was a HTTP Digest request, previous request was 401. Client needs some data sent by the server so + we should complete the ChallengeResponse. + */ + + if (ChallengeScheme.HTTP_DIGEST.equals(challengeScheme)) { + // User server's data to complete the challengeResponse object + ChallengeRequest digestChallengeRequest = retrieveDigestChallengeRequest(resource); + ChallengeResponse challengeResponse = new ChallengeResponse(digestChallengeRequest, resource.getResponse(), + username, password.toCharArray()); + testDigestRequest(expectedStatusCode, resource, challengeResponse); + } + assertEquals(expectedStatusCode, resource.getStatus().getCode()); + } + + private void testDigestRequest(int expectedStatusCode, ClientResource resource, ChallengeResponse challengeResponse){ + resource.setChallengeResponse(challengeResponse); + try { + resource.get(); + } catch (ResourceException rex) { + //Don't use it. Use the status code to check afterwards + } + assertEquals(expectedStatusCode, resource.getStatus().getCode()); + } + + private ChallengeRequest retrieveDigestChallengeRequest (ClientResource resource) { + ChallengeRequest digestChallengeRequest = null; + for (ChallengeRequest cr : resource.getChallengeRequests()) { + if (ChallengeScheme.HTTP_DIGEST.equals(cr.getScheme())) { + digestChallengeRequest = cr; + break; + } + } + return digestChallengeRequest; + } + + private void initializeSSLProperties() { + conf.set("restapi.auth.ssl.storepath", "src/test/nutch-ssl.keystore.jks"); + conf.set("restapi.auth.ssl.storepass", "password"); + conf.set("restapi.auth.ssl.keypass", "password"); + } + + /** + * Starts the server with given authentication type + * + * @param authenticationType authentication type + */ + public void startServer(AuthenticationTypeEnum authenticationType) { + conf.set("restapi.auth", authenticationType.toString()); + if(AuthenticationTypeEnum.SSL.equals(authenticationType)) { + initializeSSLProperties(); + } + + RAMConfManager ramConfManager = new RAMConfManager(NutchConfiguration.getUUID(conf), conf); + nutchServer = new NutchServer(ramConfManager, NutchConfiguration.getUUID(conf)); + nutchServer.start(); + } + + /** + * Stops server + */ + public void stopServer() { + nutchServer.stop(true); + } + +} diff --git a/src/test/org/apache/nutch/api/TestAPI.java b/src/test/org/apache/nutch/api/TestAPI.java deleted file mode 100644 index 28e999a5af..0000000000 --- a/src/test/org/apache/nutch/api/TestAPI.java +++ /dev/null @@ -1,225 +0,0 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -// CURRENTLY DISABLED. TESTS ARE FLAPPING FOR NO APPARENT REASON. -// SHALL BE FIXED OR REPLACES BY NEW API IMPLEMENTATION - -package org.apache.nutch.api; - -//import static org.junit.Assert.*; -// -//import java.util.HashMap; -//import java.util.Map; -// -//import org.apache.nutch.api.JobManager.JobType; -//import org.apache.nutch.metadata.Nutch; -//import org.apache.nutch.util.NutchTool; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -//import org.restlet.ext.jackson.JacksonRepresentation; -//import org.restlet.representation.Representation; -//import org.restlet.resource.ClientResource; - -public class TestAPI { - @Test - public void test() throws Exception { - } - // - // private static NutchServer server; - // ClientResource cli; - // - // private static String baseUrl = "http://localhost:8192/nutch/"; - // - // @BeforeClass - // public static void before() throws Exception { - // server = new NutchServer(8192); - // server.start(); - // } - // - // @AfterClass - // public static void after() throws Exception { - // if (!server.stop(false)) { - // for (int i = 1; i < 11; i++) { - // System.err.println("Waiting for jobs to complete - " + i + "s"); - // try { - // Thread.sleep(1000); - // } catch (Exception e) {}; - // server.stop(false); - // if (!server.isRunning()) { - // break; - // } - // } - // } - // if (server.isRunning()) { - // System.err.println("Forcibly stopping server..."); - // server.stop(true); - // } - // } - // - // @Test - // public void testInfoAPI() throws Exception { - // ClientResource cli = new ClientResource(baseUrl); - // String expected = - // "[[\"admin\",\"Service admin actions\"],[\"confs\",\"Configuration manager\"],[\"db\",\"DB data streaming\"],[\"jobs\",\"Job manager\"]]"; - // String got = cli.get().getText(); - // assertEquals(expected, got); - // } - // - // @SuppressWarnings("rawtypes") - // @Test - // public void testConfsAPI() throws Exception { - // ClientResource cli = new ClientResource(baseUrl + ConfResource.PATH); - // assertEquals("[\"default\"]", cli.get().getText()); - // // create - // Map map = new HashMap(); - // map.put(Params.CONF_ID, "test"); - // HashMap props = new HashMap(); - // props.put("testProp", "blurfl"); - // map.put(Params.PROPS, props); - // JacksonRepresentation> jr = - // new JacksonRepresentation>(map); - // System.out.println(cli.put(jr).getText()); - // assertEquals("[\"default\",\"test\"]", cli.get().getText()); - // cli = new ClientResource(baseUrl + ConfResource.PATH + "/test"); - // Map res = cli.get(Map.class); - // assertEquals("blurfl", res.get("testProp")); - // // delete - // cli.delete(); - // cli = new ClientResource(baseUrl + ConfResource.PATH); - // assertEquals("[\"default\"]", cli.get().getText()); - // } - // - // @SuppressWarnings("rawtypes") - // @Test - // public void testJobsAPI() throws Exception { - // ClientResource cli = new ClientResource(baseUrl + JobResource.PATH); - // assertEquals("[]", cli.get().getText()); - // // create - // Map map = new HashMap(); - // map.put(Params.JOB_TYPE, JobType.READDB.toString()); - // map.put(Params.CONF_ID, "default"); - // Representation r = cli.put(map); - // String jobId = r.getText(); - // assertNotNull(jobId); - // assertTrue(jobId.startsWith("default-READDB-")); - // // list - // Map[] list = cli.get(Map[].class); - // assertEquals(1, list.length); - // String id = (String)list[0].get("id"); - // String state = (String)list[0].get("state"); - // assertEquals(jobId, id); - // assertEquals(state, "RUNNING"); - // int cnt = 10; - // do { - // try { - // Thread.sleep(2000); - // } catch (Exception e) {}; - // list = cli.get(Map[].class); - // state = (String)list[0].get("state"); - // if (!state.equals("RUNNING")) { - // break; - // } - // } while (--cnt > 0); - // assertTrue(cnt > 0); - // if (list == null) return; - // for (Map m : list) { - // System.out.println(m); - // } - // } - // - // @SuppressWarnings("unchecked") - // @Test - // public void testStopKill() throws Exception { - // ClientResource cli = new ClientResource(baseUrl + JobResource.PATH); - // // create - // Map map = new HashMap(); - // map.put(Params.JOB_TYPE, JobType.CLASS.toString()); - // Map args = new HashMap(); - // map.put(Params.ARGS, args); - // args.put(Nutch.ARG_CLASS, SpinningJob.class.getName()); - // map.put(Params.CONF_ID, "default"); - // Representation r = cli.put(map); - // String jobId = r.getText(); - // cli.release(); - // assertNotNull(jobId); - // System.out.println(jobId); - // assertTrue(jobId.startsWith("default-CLASS-")); - // ClientResource stopCli = new ClientResource(baseUrl + JobResource.PATH + - // "?job=" + jobId + "&cmd=stop"); - // r = stopCli.get(); - // assertEquals("true", r.getText()); - // stopCli.release(); - // Thread.sleep(2000); // wait for the job to finish - // ClientResource jobCli = new ClientResource(baseUrl + JobResource.PATH + "/" - // + jobId); - // Map res = jobCli.get(Map.class); - // res = (Map)res.get("result"); - // assertEquals("stopped", res.get("res")); - // jobCli.release(); - // // restart and kill - // r = cli.put(map); - // jobId = r.getText(); - // cli.release(); - // assertNotNull(jobId); - // System.out.println(jobId); - // assertTrue(jobId.startsWith("default-CLASS-")); - // ClientResource killCli = new ClientResource(baseUrl + JobResource.PATH + - // "?job=" + jobId + "&cmd=abort"); - // r = killCli.get(); - // assertEquals("true", r.getText()); - // killCli.release(); - // Thread.sleep(2000); // wait for the job to finish - // jobCli = new ClientResource(baseUrl + JobResource.PATH + "/" + jobId); - // res = jobCli.get(Map.class); - // res = (Map)res.get("result"); - // assertEquals("killed", res.get("res")); - // jobCli.release(); - // } - // - // public static class SpinningJob extends NutchTool { - // volatile boolean shouldStop = false; - // - // @Override - // public Map run(Map args) throws Exception { - // status.put(Nutch.STAT_MESSAGE, "running"); - // int cnt = 60; - // while (!shouldStop && cnt-- > 0) { - // Thread.sleep(1000); - // } - // if (cnt == 0) { - // results.put("res", "failed"); - // } - // return results; - // } - // - // @Override - // public boolean stopJob() throws Exception { - // results.put("res", "stopped"); - // shouldStop = true; - // return true; - // } - // - // @Override - // public boolean killJob() throws Exception { - // results.put("res", "killed"); - // shouldStop = true; - // return true; - // } - // - // } -} diff --git a/src/test/org/apache/nutch/api/TestNutchAPI.java b/src/test/org/apache/nutch/api/TestNutchAPI.java new file mode 100644 index 0000000000..56aed92530 --- /dev/null +++ b/src/test/org/apache/nutch/api/TestNutchAPI.java @@ -0,0 +1,100 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.nutch.api; + +import org.apache.nutch.api.security.AuthenticationTypeEnum; +import org.junit.After; +import org.junit.Ignore; +import org.junit.Test; +import org.restlet.data.ChallengeScheme; + +/** + * Test class for {@link org.apache.nutch.api.NutchServer} + */ +public class TestNutchAPI extends AbstractNutchAPITestBase { + + /** + * Test insecure connection + */ + @Test + public void testInsecure() { + startServer(AuthenticationTypeEnum.NONE); + testRequest(200, 8081); + } + + /** + * Test Basic Authentication for invalid username/password pair, + * authorized username/password pair and insufficient privileged username/password pair + */ + @Test + public void testBasicAuth() { + startServer(AuthenticationTypeEnum.BASIC); + //Check for an invalid username/password pair + testRequest(401, 8081, "xxx", "xxx", ChallengeScheme.HTTP_BASIC); + + //Check for an authorized username/password pair + testRequest(200, 8081, "admin", "admin", ChallengeScheme.HTTP_BASIC); + + //Check for an insufficient privileged username/password pair + testRequest(403, 8081, "user", "user", ChallengeScheme.HTTP_BASIC); + } + + /** + * Test Digest Authentication for invalid username/password pair, + * authorized username/password pair and insufficient privileged username/password pair + */ + @Test + public void testDigestAuth() { + startServer(AuthenticationTypeEnum.DIGEST); + //Check for an invalid username/password pair + testRequest(401, 8081, "xxx", "xxx", ChallengeScheme.HTTP_DIGEST); + + //Check for an authorized username/password pair + testRequest(200, 8081, "admin", "admin", ChallengeScheme.HTTP_DIGEST); + + //Check for an insufficient privileged username/password pair + testRequest(403, 8081, "user", "user", ChallengeScheme.HTTP_DIGEST); + } + + /** + * Test SSL for invalid username/password pair, + * authorized username/password pair and insufficient privileged username/password pair + */ + @Ignore + @Test + public void testSSL() { + startServer(AuthenticationTypeEnum.SSL); + //Check for an invalid username/password pair + testRequest(401, 8081, "xxx", "xxx"); + + //Check for an authorized username/password pair + testRequest(200, 8081, "admin", "admin"); + + //Check for an insufficient privileged username/password pair + testRequest(403, 8081, "user", "user"); + } + + /** + * Stops the {@link NutchServer} + */ + @After + public void tearDown() { + stopServer(); + } + +} +