Merge PR snabbco#1370 (v2018.09 release) into master

.version

@@ -1 +1 @@
-2018.06
+2018.09

src/README.md

@@ -258,6 +258,10 @@ following keys are recognized:
    printed.
 
 
+— Function **engine.stop**
+
+Stop all apps in the engine by loading an empty configuration.
+
 — Function **engine.now**
 
 Returns monotonic time in seconds as a floating point number. Suitable

src/apps/bridge/README.md

@@ -9,7 +9,7 @@ arriving on a free input port may be forwarded to all other output
 ports. Packets arriving on an input port that belongs to a split-horizon
 group are never forwarded to any output port belonging to the same
 split-horizon group. There are two `bridge` implementations available:
-`apps.bridge.flooding` and apps.bridge.learning`.
+`apps.bridge.flooding` and `apps.bridge.learning`.
 
     DIAGRAM: bridge
                +----------+

src/apps/example/asm.dasl

@@ -19,7 +19,8 @@ local asm_status = ffi.new("uint32_t[1]")
 |.arch x64
 |.actionlist actions
 local Dst = dasm.new(actions)
-| mov dword [asm_status], 0xdeadbeef
+| mov64 rax, asm_status
+| mov dword [rax], 0xdeadbeef
 | ret
 code = Dst:build() -- assign to 'code' to avoid machine code being GC'd
 fptr = ffi.cast("void(*)()", code)

src/apps/ipfix/README.md

@@ -10,7 +10,9 @@ included).  The exporter can produce output in either the standard RFC
 
     DIAGRAM: IPFIX
                    +-----------+
+                   |           |
     input     ---->*   IPFIX   *---->  output
+                   |           |
                    +-----------+
 
 See the `snabb ipfix probe` command-line interface for a program built

src/apps/vhost/vhost_user.h

@@ -21,6 +21,10 @@ enum {
     VHOST_USER_SET_VRING_KICK = 12,
     VHOST_USER_SET_VRING_CALL = 13,
     VHOST_USER_SET_VRING_ERR = 14,
+    VHOST_USER_GET_PROTOCOL_FEATURES = 15,
+    VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+    VHOST_USER_GET_QUEUE_NUM = 17,
+    VHOST_USER_SET_VRING_ENABLE = 18,
     VHOST_USER_MAX
 };
 

src/apps/vhost/vhost_user.lua

@@ -111,21 +111,25 @@ end
 
 -- Table of request code -> name of handler method
 handler_names = {
-   [C.VHOST_USER_NONE]            = 'none',
-   [C.VHOST_USER_GET_FEATURES]    = 'get_features',
-   [C.VHOST_USER_SET_FEATURES]    = 'set_features',
-   [C.VHOST_USER_SET_OWNER]       = 'set_owner',
-   [C.VHOST_USER_RESET_OWNER]     = 'reset_owner',
-   [C.VHOST_USER_SET_MEM_TABLE]   = 'set_mem_table',
-   [C.VHOST_USER_SET_LOG_BASE]    = 'set_log_base',
-   [C.VHOST_USER_SET_LOG_FD]      = 'set_log_fd',
-   [C.VHOST_USER_SET_VRING_NUM]   = 'set_vring_num',
-   [C.VHOST_USER_SET_VRING_ADDR]  = 'set_vring_addr',
-   [C.VHOST_USER_SET_VRING_BASE]  = 'set_vring_base',
-   [C.VHOST_USER_GET_VRING_BASE]  = 'get_vring_base',
-   [C.VHOST_USER_SET_VRING_KICK]  = 'set_vring_kick',
-   [C.VHOST_USER_SET_VRING_CALL]  = 'set_vring_call',
-   [C.VHOST_USER_SET_VRING_ERR]   = 'set_vring_err'
+   [C.VHOST_USER_NONE]                  = 'none',
+   [C.VHOST_USER_GET_FEATURES]          = 'get_features',
+   [C.VHOST_USER_SET_FEATURES]          = 'set_features',
+   [C.VHOST_USER_SET_OWNER]             = 'set_owner',
+   [C.VHOST_USER_RESET_OWNER]           = 'reset_owner',
+   [C.VHOST_USER_SET_MEM_TABLE]         = 'set_mem_table',
+   [C.VHOST_USER_SET_LOG_BASE]          = 'set_log_base',
+   [C.VHOST_USER_SET_LOG_FD]            = 'set_log_fd',
+   [C.VHOST_USER_SET_VRING_NUM]         = 'set_vring_num',
+   [C.VHOST_USER_SET_VRING_ADDR]        = 'set_vring_addr',
+   [C.VHOST_USER_SET_VRING_BASE]        = 'set_vring_base',
+   [C.VHOST_USER_GET_VRING_BASE]        = 'get_vring_base',
+   [C.VHOST_USER_SET_VRING_KICK]        = 'set_vring_kick',
+   [C.VHOST_USER_SET_VRING_CALL]        = 'set_vring_call',
+   [C.VHOST_USER_SET_VRING_ERR]         = 'set_vring_err',
+   [C.VHOST_USER_GET_PROTOCOL_FEATURES] = 'get_protocol_features',
+   [C.VHOST_USER_SET_PROTOCOL_FEATURES] = 'set_protocol_features',
+   [C.VHOST_USER_GET_QUEUE_NUM]         = 'get_queue_num',
+   [C.VHOST_USER_SET_VRING_ENABLE]      = 'set_vring_enable'
 }
 
 -- Process all vhost_user requests from QEMU.
@@ -178,6 +182,26 @@ function VhostUser:set_features (msg)
    self.dev:set_features(features)
 end
 
+function VhostUser:get_protocol_features (msg)
+   msg.u64 = 0ULL -- no extensions supported for now
+   msg.size = ffi.sizeof("uint64_t")
+   self:reply(msg)
+end
+
+function VhostUser:set_protocol_features (msg)
+   -- ignore protocol features for now (FIXME)
+end
+
+function VhostUser:get_queue_num (msg)
+   -- ignore for now (FIXME)
+end
+
+-- Handle VHOST_USER_SET_VRING_ENABLE, which explicitly enables/disables the
+-- ring (this msg is only used if VHOST_USER_F_PROTOCOL_FEATURES is used)
+function VhostUser:set_vring_enable (msg)
+   self.vhost_ready = msg.u64 ~= 0
+end
+
 -- Feature cache: A kludge to be compatible with a "QEMU reconnect" patch.
 --
 -- QEMU upstream (circa 2015) does not support the vhost-user device

src/core/app.lua

@@ -137,6 +137,12 @@ function configure (new_config)
    counter.add(configs)
 end
 
+
+-- Stop all apps by loading an empty configuration.
+function stop ()
+   configure(config.new())
+end
+
 -- Removes the claim on a name, freeing it for other programs.
 --
 -- This relinquish a claim on a name if one exists. if the name does not
@@ -724,6 +730,11 @@ function selftest ()
    main({duration = 4, report = {showapps = true}})
    assert(app_table.app3 ~= orig_app3) -- should be restarted
 
+   -- Check engine stop
+   assert(not lib.equal(app_table, {}))
+   engine.stop()
+   assert(lib.equal(app_table, {}))
+
    -- Check one can't unclaim a name if no name is claimed.
    assert(not pcall(unclaim_name))
 

src/core/lib.lua

@@ -457,84 +457,8 @@ function root_check (message)
    end
 end
 
--- Simple token bucket for rate-limiting of events.  A token bucket is
--- created through
---
---  local tb = token_bucket_new({ rate = <rate> })
---
--- where <rate> is the maximum allowed rate in Hz, which defaults to
--- 10.  Conceptually, <rate> tokens are added to the bucket each
--- second and the bucket can hold no more than <rate> tokens but at
--- least one.
---
-
-local token_bucket = {}
-token_bucket.mt = { __index = token_bucket }
-token_bucket.default = { rate = 10 }
-function token_bucket_new (config)
-   local config = config or token_bucket.default
-   local tb = setmetatable({}, token_bucket.mt)
-   tb:rate(config.rate or token_bucket.default.rate)
-   tb._tstamp = C.get_monotonic_time()
-   return tb
-end
-
--- The rate can be set with the rate() method at any time, which fills
--- the token bucket an also returns the previous value.  If called
--- with a nil argument, returns the currently configured rate.
-function token_bucket:rate (rate)
-   if rate ~= nil then
-      local old_rate = self._rate
-      self._rate = rate
-      self._max_tokens = math.max(rate, 1)
-      self._tokens = self._max_tokens
-      return old_rate
-   end
-   return self._rate
-end
-
-function token_bucket:_update (tokens)
-   local now = C.get_monotonic_time()
-   local tokens = math.min(self._max_tokens, tokens + self._rate*(now-self._tstamp))
-   self._tstamp = now
-   return tokens
-end
-
--- The take() method tries to remove <n> tokens from the bucket.  If
--- enough tokens are available, they are subtracted from the bucket
--- and a true value is returned.  Otherwise, the bucket remains
--- unchanged and a false value is returned.  For efficiency, the
--- tokens accumulated since the last call to take() or can_take() are
--- only added if the request can not be fulfilled by the state of the
--- bucket when the method is called.
-function token_bucket:take (n)
-   local n = n or 1
-   local result = false
-   local tokens = self._tokens
-   if n > tokens then
-      tokens = self:_update(tokens)
-   end
-   if n <= tokens then
-      tokens = tokens - n
-      result = true
-   end
-   self._tokens = tokens
-   return result
-end
-
--- The can_take() method returns a true value if the bucket contains
--- at least <n> tokens, false otherwise.  The bucket is updated in a
--- layz fashion as described for the take() method.
-function token_bucket:can_take (n)
-   local n = n or 1
-   local tokens = self._tokens
-   if n <= tokens then
-      return true
-   end
-   tokens = self:_update(tokens)
-   self._tokens = tokens
-   return n <= tokens
-end
+-- Backward compatibility
+token_bucket_new = require("lib.token_bucket").new
 
 -- Simple rate-limited logging facility.  Usage:
 --

src/core/memory.lua

@@ -146,12 +146,12 @@ function allocate_huge_page (size,  persistent)
    local fd = syscall.open(tmpfile, "creat, rdwr", "RWXU")
    assert(fd, "create hugetlb")
    assert(syscall.ftruncate(fd, size), "ftruncate")
-   local tmpptr = syscall.mmap(nil, size, "read, write", "shared, hugetlb", fd, 0)
+   local tmpptr = syscall.mmap(nil, size, "read, write", "shared", fd, 0)
    assert(tmpptr, "mmap hugetlb")
    assert(syscall.mlock(tmpptr, size))
    local phys = resolve_physical(tmpptr)
    local virt = bit.bor(phys, tag)
-   local ptr = syscall.mmap(virt, size, "read, write", "shared, hugetlb, fixed", fd, 0)
+   local ptr = syscall.mmap(virt, size, "read, write", "shared, fixed", fd, 0)
    local filename = ("/var/run/snabb/hugetlbfs/%012x.dma"):format(tonumber(phys))
    if persistent then
       assert(syscall.rename(tmpfile, filename))
@@ -160,6 +160,7 @@ function allocate_huge_page (size,  persistent)
    else
       assert(syscall.unlink(tmpfile))
    end
+   syscall.munmap(tmpptr, size)
    syscall.close(fd)
    return ptr, filename
 end

src/lib/README.ctable.md

@@ -52,6 +52,9 @@ Optional entries that may be present in the *parameters* table include:
    2.  Defaults to 0.9, for a 90% maximum occupancy ratio.
  * `min_occupancy_rate`: Minimum ratio of `occupancy/size`.  Removing an
    entry from an "empty" table will shrink the table.
+ * `resize_callback`: An optional function that is called after the
+   table has been resized.  The function is called with two arguments:
+   the ctable object and the old size. By default, no callback is used.
 
 — Function **ctable.load** *stream* *parameters*
 

src/lib/README.token_bucket.md

@@ -0,0 +1,70 @@
+### Token Bucket (lib.token_bucket)
+
+This module implements a [token
+bucket](https://en.wikipedia.org/wiki/Token_bucket) for rate-limiting
+of arbitrary events.  The bucket is filled with tokens at a constant
+rate up to a given maximum called the *burst_size*.  Tokens are added
+and removed in integer quantities.  An event can only take place if at
+least one token is available.  A burst of back-to-back events is
+allowed to happen by consuming all available tokens at a given point
+in time.  The maximum size of such a burst is determined by the
+capacity of the bucket, hence the name *burst_size*.
+
+The token bucket is updated in a lazy fashion, i.e. only when a
+request for tokens cannot be satisfied immediately.
+
+By default, a token bucket uses the `rdtsc` time source via the
+[`tsc`](./README.tsc.md) module to minimise overhead.  To override,
+the `default_source` parameter of the `tsc` module must be set
+to the desired value.
+
+#### Functions
+
+— Function **new** *config*
+
+Creates an instance of a token bucket.  The required *config* argument
+must be a table with the following keys.
+
+— Key **rate**
+
+*Required*.  The rate in units of Hz at which tokens are placed in the
+bucket as an arbitrary floating point number larger than zero.
+
+— Key **burst_size**
+
+*Optional*.  The maximum number of tokens that can be stored in the
+bucket.  The default is **rate** tokens, i.e. the amount of tokens
+accumulated over one second rounded up to the next integer.
+
+#### Methods
+
+The object returned by the **new** function provides the following
+methods.
+
+— Method **token_bucket:set** [*rate*], [*burst_size*]
+
+Set the rate and burst size to the values *rate* and *burst_size*,
+respectively, and fill the bucket to capacity.  If *rate* is `nil`,
+the rate remains unchanged.  If *burst_size* is `nil`, the burst size
+is set to the number of tokens that will be accumulated over one
+second with the new rate (like in the **new** function).
+
+— Method **token_bucket:get**
+
+Returns the current rate and burst size.
+
+— Method **token_bucket:can_take** [*n*]
+
+Returns `true` if at least *n* tokens are available, `false`
+otherwise.  If *n* is `nil`, the bucket is checked for a single token.
+
+— Method **token_bucket:take** [*n*]
+
+If at least *n* tokens are available, they are removed from the bucket
+and the method returns `true`.  Otherwise, the bucket remains
+unchanged and `false` is returned. If *n* is `nil`, the bucket is
+checked for a single token.
+
+— Method **token_bucket:take_burst**
+
+Takes all available tokens from the bucket and returns that number.