Merge pull request #5470 from brenton/BZ1064631

Merged by openshift-bot

node/conf/node.conf

@@ -167,3 +167,29 @@ OPENSHIFT_FRONTEND_HTTP_PLUGINS=openshift-origin-frontend-apache-mod-rewrite,ope
 # Defaults to 2000 bytes.
 #
 # METRICS_MAX_LINE_LENGTH=2000
+
+# Each Gear has its own dedicated loopback IP address range that is determined
+# by its UID.  Valid UIDs are in the range of 0-2^32 however there are only
+# 65535 * 2 loopback IP address ranges available for OpenShift Gears.  This is
+# never a problem because a single district will never exhaust this many
+# loopback IPs.
+#
+# However, in environments where high numbered UIDs are required it's important
+# 127.0.0.1 is never given to a gear that "wraps around". To avoid this
+# situation an offset is configured.  By default any UID greater than or equal
+# to WRAPAROUND_UID will offset by IP_ADDRESS_WRAPAROUND_OFFSET and then
+# shifted left 7 bits.  For example, setting IP_ADDRESS_WRAPAROUND_OFFSET=1
+# will cause the wrap around IP addresses to start at 127.0.0.128.  The left
+# shifting by 7 is important because each Gear has an IP address range of 127
+# IPs (127.x.x.0 and 127.x.x.128 are unused).  These settings must be the same
+# for all Nodes in a district and must not be changed for any Node with running
+# Gears.
+#
+# WRAPAROUND_UID is also used for Traffic Control where OpenShift manages a
+# maximum of 65535 queuing disciplines.  Again, this is never a problem in a
+# District however we need the queuing disciplines to wrap around.
+#
+# Defaults:
+#
+# WRAPAROUND_UID=65536
+# WRAPAROUND_IP_ADDRESS_OFFSET=1

node/lib/openshift-origin-node/utils/cgroups/libcgroup.rb

@@ -50,6 +50,8 @@ def initialize(uuid)
 
             @cgroup_root = self.class.cgroup_root
             @cgroup_path = "#{@cgroup_root}/#{@uuid}"
+            @config      = OpenShift::Config.new
+            @wrap_around_uid = (@config.get("WRAPAROUND_UID") || 65536).to_i
           end
 
           def self.cgroup_root
@@ -321,13 +323,13 @@ def processes
           # Compute the network class id
           # Major = 1
           # Minor = UID
-          # Caveat: 0 <= Minor <= 0xFFFF (65535)
+          # Caveat: 0 <= Minor
           def net_cls
             major = 1
-            if (uid < 1) or (uid > 0xFFFF)
+            if (uid < 1)
               raise RuntimeError, "Cannot assign network class id for: #{uid}"
             end
-            (major << 16) + uid
+            (major << 16) + (uid % @wrap_around_uid)
           end
 
 

node/lib/openshift-origin-node/utils/tc.rb

@@ -45,6 +45,7 @@ def initialize
           @output = []
 
           @config    = ::OpenShift::Config.new('/etc/openshift/node.conf')
+          @wrap_around_uid = (@config.get("WRAPAROUND_UID") || 65536).to_i
           @resources = ::OpenShift::Runtime::Utils::Cgroups::Config.new('/etc/openshift/resource_limits.conf')
 
 
@@ -115,15 +116,15 @@ def get_interface_mtu(iface)
         def parse_valid_user(uuid)
           pwent = Etc.getpwnam(uuid.to_s)
           if block_given?
-            yield(pwent, pwent.uid.to_s(16))
+            yield(pwent, (pwent.uid % @wrap_around_uid).to_s(16))
           end
         end
 
         def with_all_users
           users=[]
           Etc.passwd do |pwent|
             if pwent.gecos == (@config.get('GEAR_GECOS') or "OO guest")
-              users << [pwent.name, pwent, pwent.uid.to_s(16)]
+              users << [pwent.name, pwent, (pwent.uid % @wrap_around_uid).to_s(16)]
             end
           end
           if block_given?

node/test/unit/application_container_test.rb

@@ -258,8 +258,35 @@ def test_get_ip_addr_success
           [501, 10, "127.0.250.138"],
           [501, 20, "127.0.250.148"],
           [501, 100, "127.0.250.228"],
+          [501, 127, "127.0.250.255"],
           [540, 1, "127.1.14.1"],
-          [560, 7, "127.1.24.7"]
+          [560, 7, "127.1.24.7"],
+
+          # 2x 2^16 +- 1
+          [131071, 1, "127.128.0.1"],
+          [131071, 10, "127.128.0.10"],
+          [131071, 127, "127.128.0.127"],
+
+          [131072, 1, "127.0.0.129"],
+          [131072, 10, "127.0.0.138"],
+          [131072, 127, "127.0.0.255"],
+
+          [131073, 1, "127.0.1.1"],
+          [131073, 10, "127.0.1.10"],
+          [131073, 127, "127.0.1.127"],
+
+          # 10x 2^16 +- 1
+          [655359, 1, "127.128.0.1"],
+          [655359, 10, "127.128.0.10"],
+          [655359, 127, "127.128.0.127"],
+
+          [655360, 1, "127.0.0.129"],
+          [655360, 10, "127.0.0.138"],
+          [655360, 127, "127.0.0.255"],
+
+          [655361, 1, "127.0.1.1"],
+          [655361, 10, "127.0.1.10"],
+          [655361, 127, "127.0.1.127"],
       ]
 
       scenarios.each do |s|
@@ -275,7 +302,7 @@ def test_get_ip_addr_success
         container = OpenShift::Runtime::ApplicationContainer.new("gear_uuid", "gear_uuid", s[0],
                                                                  "app_name", "gear_uuid", "namespace", nil, nil, nil)
 
-        assert_equal container.get_ip_addr(s[1]), s[2]
+        assert_equal s[2], container.get_ip_addr(s[1])
       end
     end
 

plugins/container/selinux/lib/openshift/runtime/containerization/selinux_container.rb

@@ -19,6 +19,12 @@ def initialize(application_container)
           @config     = ::OpenShift::Config.new
           @gear_shell = @config.get("GEAR_SHELL")    || "/bin/bash"
           @traffic_control_enabled = @config.get_bool("TRAFFIC_CONTROL_ENABLED", "true")
+          @wrap_around_uid = (@config.get("WRAPAROUND_UID") || 65536).to_i
+          @ip_offset = (@config.get("IP_ADDRESS_WRAPAROUND_OFFSET") || 1).to_i
+
+          if @ip_offset >= 1000
+            raise "IP_ADDRESS_WRAPAROUND_OFFSET should be less than #{1000}"
+          end
         end
 
         # Public
@@ -203,22 +209,30 @@ def start
         # host identifier (LSB of the IP). The host identifier must be a value between 1-127
         # inclusive.
         #
-        # The global user IP range begins at 0x7F000000.
+        # The global user IP range begins at 0x7F000000 (127.0.0.0) for all
+        # UIDs under 65536.  For UIDs over that the range begins at
+        # 127.0.0.0 + (WRAPAROUND_IP_ADDRESS_OFFSET << 7)
         #
         # Returns an IP address string in dotted-quad notation.
         def get_ip_addr(host_id)
           raise "Invalid host_id specified" unless host_id && host_id.is_a?(Integer)
 
-          if @container.uid.to_i < 0 || @container.uid.to_i > 262143
-            raise "User uid #{@container.uid} is outside the working range 0-262143"
+          if @container.uid.to_i < 0 || @container.uid.to_i > (2 << 31)
+            raise "User uid #{@container.uid} must be unsigned 32 bit integers."
           end
 
           if host_id < 1 || host_id > 127
             raise "Supplied host identifier #{host_id} must be between 1 and 127"
           end
 
+          # Can't do this in the constructor because sometimes #uid isn't set.
+          if @container.uid.to_i < @wrap_around_uid
+            @ip_offset = 0
+          end
+
           # Generate an IP (32-bit unsigned) in the user's range
-          ip = 0x7F000000 + (@container.uid.to_i << 7)  + host_id
+          loopback_start = 0x7F000000
+          ip = loopback_start + (@ip_offset << 7) + ((@container.uid.to_i % @wrap_around_uid) << 7) + host_id
 
           # Return the IP in dotted-quad notation
           "#{ip >> 24}.#{ip >> 16 & 0xFF}.#{ip >> 8 & 0xFF}.#{ip & 0xFF}"