diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java index c83afc7fe4b92..d4758dbfdad97 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java @@ -126,12 +126,13 @@ public void authorize(UserGroupInformation user, } if (addr != null) { String hostAddress = addr.getHostAddress(); - if (hosts.length != 2 || !hosts[0].includes(hostAddress) || - hosts[1].includes(hostAddress)) { + if (hosts.length != 2 || + !hosts[0].includes(hostAddress, user.getUserName()) || + hosts[1].includes(hostAddress, user.getUserName())) { AUDITLOG.warn(AUTHZ_FAILED_FOR + " for protocol=" + protocol + " from host = " + hostAddress); throw new AuthorizationException("Host " + hostAddress + - " is not authorized for protocol " + protocol) ; + " is not authorized for protocol " + protocol); } } AUDITLOG.info(AUTHZ_SUCCESSFUL_FOR + user + " for protocol="+protocol); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java index f87d059dec75b..a0ae245506a6e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java @@ -20,21 +20,18 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; import java.util.Set; -import org.apache.commons.net.util.SubnetUtils; - import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Container class which holds a list of ip/host addresses and + * Container class which holds a list of ip/host addresses and * answers membership queries. * * Accepts list of ip addresses, ip addreses in CIDR format and/or @@ -42,32 +39,28 @@ */ public class MachineList { - + public static final Logger LOG = LoggerFactory.getLogger(MachineList.class); public static final String WILDCARD_VALUE = "*"; /** * InetAddressFactory is used to obtain InetAddress from host. * This class makes it easy to simulate host to ip mappings during testing. - * */ public static class InetAddressFactory { static final InetAddressFactory S_INSTANCE = new InetAddressFactory(); - public InetAddress getByName (String host) throws UnknownHostException { + public InetAddress getByName(String host) throws UnknownHostException { return InetAddress.getByName(host); } } private final boolean all; - private final Set inetAddresses; + private TrieTree node; private final Collection entries; - private final List cidrAddresses; - private final InetAddressFactory addressFactory; /** - * * @param hostEntries comma separated ip/cidr/host addresses */ public MachineList(String hostEntries) { @@ -79,7 +72,6 @@ public MachineList(String hostEntries, InetAddressFactory addressFactory) { } /** - * * @param hostEntries collection of separated ip/cidr/host addresses */ public MachineList(Collection hostEntries) { @@ -87,82 +79,121 @@ public MachineList(Collection hostEntries) { } /** - * Accepts a collection of ip/cidr/host addresses - * + * Accepts a collection of ip/cidr/host addresses. + * * @param hostEntries * @param addressFactory addressFactory to convert host to InetAddress */ public MachineList(Collection hostEntries, - InetAddressFactory addressFactory) { - this.addressFactory = addressFactory; + InetAddressFactory addressFactory) { if (hostEntries != null) { entries = new ArrayList<>(hostEntries); if ((hostEntries.size() == 1) && (hostEntries.contains(WILDCARD_VALUE))) { all = true; - inetAddresses = null; - cidrAddresses = null; + node = null; } else { all = false; - Set addrs = new HashSet<>(); - List cidrs = new LinkedList(); + node = new TrieTree(); for (String hostEntry : hostEntries) { - //ip address range - if (hostEntry.indexOf("/") > -1) { - try { - SubnetUtils subnet = new SubnetUtils(hostEntry); - subnet.setInclusiveHostCount(true); - cidrs.add(subnet.getInfo()); - } catch (IllegalArgumentException e) { - LOG.warn("Invalid CIDR syntax : " + hostEntry); - throw e; + String[] splits = hostEntry.split(":"); + HashSet userSet = null; + if (splits.length == 2) { + userSet = new HashSet<>(); + String[] userArray = splits[1].split("\\|"); + if (userArray.length > 0) { + for (String u : userArray) { + String user = u.trim(); + if (user.length() > 0) { + userSet.add(user); + } + } } - } else { - try { - addrs.add(addressFactory.getByName(hostEntry)); - } catch (UnknownHostException e) { - LOG.warn(e.toString()); + } + String host = splits[0]; + String binaryIp; + try { + if (host.contains("/")) { + String[] cidrArray = host.split("/"); + String ip = cidrArray[0]; + int mask = Integer.parseInt(cidrArray[1]); + if (!isValidIPv4(ip)) { + LOG.warn("Invalid CIDR syntax : " + hostEntry); + throw new IllegalArgumentException(); + } + binaryIp = subtractBinaryNumber(ipToBinaryNumber(ip), mask); + } else { + binaryIp = ipToBinaryNumber( + addressFactory.getByName(host).getHostAddress()); } + } catch (UnknownHostException e) { + LOG.warn(e.toString()); + continue; } + node.insert(binaryIp, userSet); } - inetAddresses = (addrs.size() > 0) ? addrs : null; - cidrAddresses = (cidrs.size() > 0) ? cidrs : null; + node = node.isEmpty() ? null : node; } } else { all = false; - inetAddresses = null; - cidrAddresses = null; + node = null; entries = Collections.emptyList(); } } + /** - * Accepts an ip address and return true if ipAddress is in the list. - * {@link #includes(InetAddress)} should be preferred - * to avoid possibly re-resolving the ip address. - * + * Accepts an ip address and return true if ipAddress is in the TrieTree. * @param ipAddress - * @return true if ipAddress is part of the list + * @return true if ipAddress is part is in the TrieTree. */ public boolean includes(String ipAddress) { - + if (all) { return true; } - + if (ipAddress == null) { throw new IllegalArgumentException("ipAddress is null."); } + // iterate through the ip ranges for inclusion + if (node != null) { + String binaryIp = ipToBinaryNumber(ipAddress); + if (node.search(binaryIp, null)) { + return true; + } + } + return false; + } - try { - return includes(addressFactory.getByName(ipAddress)); - } catch (UnknownHostException e) { - return false; + /** + * Accept an inet address and a user and + * return true if address and user are in the TrieTree . + * + * @param ipAddress A address. + * @return true if address is part of the TrieTree. + */ + public boolean includes(String ipAddress, String user) { + if (all) { + return true; + } + if (ipAddress == null) { + throw new IllegalArgumentException("ipAddress is null."); } + // iterate through the ip ranges for inclusion + if (node != null) { + String binaryIp = ipToBinaryNumber(ipAddress); + if (node.search(binaryIp, user)) { + return true; + } + } + return false; } /** - * Accepts an inet address and return true if address is in the list. - * @param address - * @return true if address is part of the list + * Accept an inet address and return true + * if address is in the TrieTree. + * + * @param address A InetAddress. + * @return true if address is part of the TrieTree. */ public boolean includes(InetAddress address) { if (all) { @@ -171,20 +202,40 @@ public boolean includes(InetAddress address) { if (address == null) { throw new IllegalArgumentException("address is null."); } - if (inetAddresses != null && inetAddresses.contains(address)) { + // iterate through the ip ranges for inclusion + if (node != null) { + String binaryIp = ipToBinaryNumber(address.getHostAddress()); + if (node.search(binaryIp, null)) { + return true; + } + } + return false; + } + + /** + * Accept an inet address and a user and + * return true if address and user are in the TrieTree . + * + * @param address A address. + * @param user A user. + * @return true if address is part of the TrieTree + */ + public boolean includes(InetAddress address, String user) { + if (all) { return true; } - // iterate through the ip ranges for inclusion - if (cidrAddresses != null) { - String ipAddress = address.getHostAddress(); - for(SubnetUtils.SubnetInfo cidrAddress : cidrAddresses) { - if(cidrAddress.isInRange(ipAddress)) { - return true; - } + if (address == null) { + throw new IllegalArgumentException("address is null."); + } + if (node != null) { + String binaryIp = ipToBinaryNumber(address.getHostAddress()); + if (node.search(binaryIp, user)) { + return true; } } return false; } + /** * returns the contents of the MachineList as a Collection<String> . * This can be used for testing . @@ -195,4 +246,139 @@ public boolean includes(InetAddress address) { public Collection getCollection() { return entries; } + + final private static class TrieTree { + private Node root; + + final private static class Node { + private boolean netRange; + private HashSet userSet; + private Node[] children; + + private Node() { + netRange = false; + children = new Node[2]; + Arrays.fill(children, null); + } + } + + private TrieTree() { + root = new Node(); + } + + private void insert(String ipAddress, HashSet user) { + Node currNode = root; + for (int i = 0; i < ipAddress.length(); i++) { + int index = (int) ipAddress.charAt(i) - '0'; + if (currNode.children[index] == null) { + currNode.children[index] = new Node(); + } + currNode = currNode.children[index]; + } + currNode.netRange = true; + currNode.userSet = user; + } + + private boolean isEmpty() { + return root.children[0] == null && + root.children[1] == null; + } + + private boolean search(String binaryIP, String user) { + Node currNode = root; + for (int i = 0; !currNode.netRange; i++) { + int index = (int) binaryIP.charAt(i) - '0'; + if (currNode.children[index] == null) { + return false; + } + Node nodeTmp = currNode.children[index]; + Set users = nodeTmp.userSet; + if (nodeTmp.netRange) { + if (user == null) { + return true; + } else if (users == null) { + return false; + } else if (users.contains(user) || users.contains(WILDCARD_VALUE)) { + return true; + } + } + currNode = currNode.children[index]; + } + return false; + } + } + + /** + * Get ip address binary mask. + * + * @param ipAddress A ip. + * @return A binary mask. + */ + private static String ipToBinaryNumber(String ipAddress) { + String[] octetArray = ipAddress.split("\\."); + StringBuilder binaryNumber = new StringBuilder(); + for (String str : octetArray) { + int octet = Integer.parseInt(str, 10); + StringBuilder binaryOctet = + new StringBuilder(Integer.toBinaryString(octet)); + int length = binaryOctet.length(); + if (length < 8) { + for (int i = 0; i < 8 - length; i++) { + binaryOctet.insert(0, '0'); + } + } + binaryNumber.append(binaryOctet); + } + return binaryNumber.toString(); + } + + private String subtractBinaryNumber(String s, int size) { + return s.substring(0, size); + } + + /** + * Accept an String to determine whether it is an ip address. + * + * @param ip A ip + * @return if the given string is a Ipv4 false otherwise. + */ + private static boolean isValidIPv4(String ip) { + if (ip.length() < 7) { + return false; + } + if (ip.charAt(0) == '.') { + return false; + } + if (ip.charAt(ip.length() - 1) == '.') { + return false; + } + String[] tokens = ip.split("\\."); + if (tokens.length != 4) { + return false; + } + for (String token : tokens) { + if (!isValidIPv4Segment(token)) { + return false; + } + } + return true; + } + + private static boolean isValidIPv4Segment(String token) { + if (token.startsWith("0") && token.length() > 1) { + return false; + } + try { + int parsedInt = Integer.parseInt(token); + if (parsedInt < 0 || parsedInt > 255) { + return false; + } + if (parsedInt == 0 && token.charAt(0) != '0') { + return false; + } + } catch (NumberFormatException nfe) { + return false; + } + return true; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java index d02fe604d79e3..25e19eab36d1e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java @@ -41,9 +41,10 @@ public class TestServiceAuthorization { private static final String ADDRESS = "0.0.0.0"; private static final String HOST_CONFIG = "test.protocol.hosts"; private static final String BLOCKED_HOST_CONFIG = "test.protocol.hosts.blocked"; - private static final String AUTHORIZED_IP = "1.2.3.4"; - private static final String UNAUTHORIZED_IP = "1.2.3.5"; - private static final String IP_RANGE = "10.222.0.0/16,10.113.221.221"; + private static final String AUTHORIZED_IP = "192.168.1.1"; + private static final String UNAUTHORIZED_IP = "192.168.1.2"; + private static final String IP_RANGE = + "10.222.0.0/16:drwho@EXAMPLE.COM,10.113.221.221:drwho@EXAMPLE.COM"; public interface TestProtocol1 extends TestProtocol {}; @@ -246,7 +247,7 @@ public void testMachineList() throws UnknownHostException { ServiceAuthorizationManager serviceAuthorizationManager = new ServiceAuthorizationManager(); Configuration conf = new Configuration (); - conf.set(HOST_CONFIG, "1.2.3.4"); + conf.set(HOST_CONFIG, "192.168.1.1:drwho@EXAMPLE.COM"); serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); try { serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf, @@ -360,7 +361,7 @@ public void testDefaultBlockedMachineList() throws UnknownHostException { conf.set( "security.service.authorization.default.hosts.blocked", IP_RANGE); - conf.set(BLOCKED_HOST_CONFIG, "1.2.3.4"); + conf.set(BLOCKED_HOST_CONFIG, "192.168.1.1:drwho@EXAMPLE.COM"); serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); // TestProtocol can be accessed from "10.222.0.0" because it blocks only "1.2.3.4" try { @@ -372,7 +373,7 @@ public void testDefaultBlockedMachineList() throws UnknownHostException { // TestProtocol cannot be accessed from "1.2.3.4" try { serviceAuthorizationManager.authorize(drwho, - TestProtocol.class, conf, InetAddress.getByName("1.2.3.4")); + TestProtocol.class, conf, InetAddress.getByName("192.168.1.1")); fail(); } catch (AuthorizationException e) { //expects Exception @@ -380,7 +381,7 @@ public void testDefaultBlockedMachineList() throws UnknownHostException { // TestProtocol1 can be accessed from "1.2.3.4" because it uses default block list try { serviceAuthorizationManager.authorize(drwho, - TestProtocol1.class, conf, InetAddress.getByName("1.2.3.4")); + TestProtocol1.class, conf, InetAddress.getByName("192.168.1.1")); } catch (AuthorizationException e) { fail(); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java index e54656a1cc62a..a3bedc4b4bdcd 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java @@ -42,8 +42,12 @@ public class TestMachineList { private static String IP_CIDR_LIST = "10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24"; private static String HOST_LIST = "host1,host4"; + private static String HOSTNAME_IP_USER_CIDR_LIST = + "host1:user1|user2,10.222.0.0/16:user1|" + + "user2,10.119.103.110:*,10.119.103.111"; private static String HOSTNAME_IP_CIDR_LIST = - "host1,10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24,host4,"; + "host1,10.222.0.0/16,10.119.103.110," + + "10.119.103.112,10.119.103.114,10.241.23.0/24,host4,"; class TestAddressFactory extends MachineList.InetAddressFactory { private Map cache = new HashMap<>(); @@ -148,18 +152,18 @@ public void testHostNames() throws UnknownHostException { public void testHostNamesReverserIpMatch() throws UnknownHostException { // create MachineList with a list of of Hostnames TestAddressFactory addressFactory = new TestAddressFactory(); - addressFactory.put("1.2.3.1", "host1"); - addressFactory.put("1.2.3.4", "host4"); - addressFactory.put("1.2.3.5", "host5"); + addressFactory.put("192.168.1.1", "host1"); + addressFactory.put("192.168.1.4", "host4"); + addressFactory.put("192.168.1.5", "host5"); MachineList ml = new MachineList( StringUtils.getTrimmedStringCollection(HOST_LIST), addressFactory ); //test for inclusion with an known IP - assertTrue(ml.includes("1.2.3.4")); + assertTrue(ml.includes("192.168.1.4")); //test for exclusion with an unknown IP - assertFalse(ml.includes("1.2.3.5")); + assertFalse(ml.includes("192.168.1.5")); } @Test @@ -215,6 +219,27 @@ public void testCIDRWith16bitmask() { assertFalse(ml.includes("10.119.103.111")); } + @Test + public void testIpWithUser() throws UnknownHostException { + TestAddressFactory addressFactory = new TestAddressFactory(); + addressFactory.put("192.168.1.2", "host1"); + //create MachineList with a list of of ip ranges specified in CIDR format + MachineList ml = + new MachineList(HOSTNAME_IP_USER_CIDR_LIST, addressFactory); + // test IPHost + assertTrue(ml.includes("192.168.1.2", "user1")); + assertFalse(ml.includes("192.168.1.2", "user3")); + assertTrue(ml.includes("192.168.1.2", null)); + // test net ranges + assertTrue(ml.includes("10.222.0.1", "user1")); + assertFalse(ml.includes("10.222.0.1", "user3")); + assertTrue(ml.includes("10.222.0.1", null)); + // test user list * + assertTrue(ml.includes("10.119.103.110", null)); + assertTrue(ml.includes("10.119.103.110", "user3")); + // test user list is null + assertFalse(ml.includes("10.119.103.111", "user3")); + } @Test public void testCIDRWith8BitMask() { //create MachineList with a list of of ip ranges specified in CIDR format