Merge pull request #11 from gconnell/seccomp

Introduce seccomp sandboxing to restrict syscalls made from stenotype after it has set up its initial high-privilege sockets.

config/config.go

@@ -68,13 +68,13 @@ func (c Config) Directory() (_ *Directory, returnedErr error) {
 	for i, thread := range c.Threads {
 		if _, err := os.Stat(thread.PacketsDirectory); err != nil {
 			return nil, fmt.Errorf("invalid packets directory %q in configuration: %v", thread.PacketsDirectory, err)
-		} else if err := os.Symlink(thread.PacketsDirectory, filepath.Join(dirname, strconv.Itoa(i))); err != nil {
+		} else if err := os.Symlink(thread.PacketsDirectory, filepath.Join(dirname, "PKT"+strconv.Itoa(i))); err != nil {
 			return nil, fmt.Errorf("couldn't create symlink for thread %d to directory %q: %v", i, thread.PacketsDirectory, err)
 		}
-		if thread.IndexDirectory != "" {
-			if err := os.Symlink(thread.IndexDirectory, filepath.Join(dirname, strconv.Itoa(i), "INDEX")); err != nil {
-				return nil, fmt.Errorf("couldn't create symlink for thread %d index to directory %q: %v", i, thread.IndexDirectory, err)
-			}
+		if _, err := os.Stat(thread.IndexDirectory); err != nil {
+			return nil, fmt.Errorf("invalid index directory %q in configuration: %v", thread.IndexDirectory, err)
+		} else if err := os.Symlink(thread.IndexDirectory, filepath.Join(dirname, "IDX"+strconv.Itoa(i))); err != nil {
+			return nil, fmt.Errorf("couldn't create symlink for index %d to directory %q: %v", i, thread.IndexDirectory, err)
 		}
 	}
 	return newDirectory(dirname, len(c.Threads)), nil
@@ -136,7 +136,7 @@ func (d *Directory) checkForNewFiles() {
 	defer d.mu.Unlock()
 	gotNew := false
 	for i := 0; i < d.threads; i++ {
-		dirpath := filepath.Join(d.name, strconv.Itoa(i))
+		dirpath := filepath.Join(d.name, "PKT" + strconv.Itoa(i))
 		files, err := ioutil.ReadDir(dirpath)
 		if err != nil {
 			log.Printf("could not read dir %q: %v", dirpath, err)

format.sh

@@ -13,4 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-find "`dirname $0`" -iname \*.cc -o -iname \*.h -execdir clang-format-3.5 -i -style=Google {} \;
+cd $(dirname $0)
+clang-format-3.5 -style=Google -i stenotype/*.{cc,h}

indexfile/indexfile.go

@@ -22,8 +22,8 @@ import (
 	"fmt"
 	"io"
 	"net"
-	"path/filepath"
 	"sort"
+	"strings"
 
 	"github.com/google/stenographer/base"
 	"github.com/google/stenographer/sstable"
@@ -39,7 +39,7 @@ type IndexFile struct {
 // IndexPathFromPath returns the path to an index file based on the path to a
 // block file.
 func IndexPathFromPath(p string) string {
-	return filepath.Join(filepath.Dir(p), "INDEX", filepath.Base(p))
+	return strings.Replace(p, "PKT", "IDX", 1)
 }
 
 // NewIndexFile returns a new handle to the named index file.

sample_config

@@ -1,6 +1,7 @@
 {
   "Threads": [
-    { "PacketsDirectory": "/tmp2/steno" }
+    { "PacketsDirectory": "/tmp2/steno",
+      "IndexDirectory": "/tmp2/stenoidx" }
   ]
   , "StenotypePath": "stenotype/stenotype"
   , "Interface": "em1"

stenotype/aio.cc

@@ -82,7 +82,7 @@ Error PWrite::Done(io_event* event) {
   long bytes_written = static_cast<long>(event->res);
   Error result;
   if (bytes_written < 0) {
-    result = AIOErrno(bytes_written);
+    result = NegErrno(bytes_written);
   } else if (bytes_written < int64_t(block.Data().size())) {
     result = ERROR("write truncated");
   }
@@ -108,19 +108,18 @@ void SingleFile::Write(io_context_t ctx, Block* b) {
     if (ret != 0 && ret != -EAGAIN) break;
     SleepForSeconds(0.001);
   }
-  CHECK_SUCCESS(AIOErrno(ret));
+  CHECK_SUCCESS(NegErrno(ret));
 }
 
 Error SingleFile::Close() {
   CHECK(Closable());
   LOG(INFO) << "Closing " << hidden_name_ << " (" << fd_ << "), truncating to "
             << (truncate_ >> 20) << "MB and moving to " << unhidden_name_;
-  LOG_IF_ERROR(Errno(ftruncate(fd_, truncate_) >= 0), "ftruncate");
-  RETURN_IF_ERROR(Errno(close(fd_) >= 0), "close");
+  LOG_IF_ERROR(Errno(ftruncate(fd_, truncate_)), "ftruncate");
+  RETURN_IF_ERROR(Errno(close(fd_)), "close");
   fd_ = 0;
-  RETURN_IF_ERROR(
-      Errno(0 == rename(hidden_name_.c_str(), unhidden_name_.c_str())),
-      "rename");
+  RETURN_IF_ERROR(Errno(rename(hidden_name_.c_str(), unhidden_name_.c_str())),
+                  "rename");
   return SUCCESS;
 }
 
@@ -146,7 +145,7 @@ Error Output::SetUp() {
     SleepForSeconds(1);
     LOG(V1) << "io_setup retrying";
   }
-  return AIOErrno(ret);
+  return NegErrno(ret);
 }
 
 Error Output::CheckForCompletedOps(bool block) {
@@ -158,7 +157,7 @@ Error Output::CheckForCompletedOps(bool block) {
     // setting errno and returning -1.
     ret = io_getevents(ctx_, block ? 1 : 0, 4, events, NULL);
   } while (ret == -EINTR);
-  RETURN_IF_ERROR(AIOErrno(ret), "io_getevents");
+  RETURN_IF_ERROR(NegErrno(ret), "io_getevents");
   for (int i = 0; i < ret; i++) {
     auto aio = reinterpret_cast<io::PWrite*>(events[i].obj->data);
     auto file = aio->file;

stenotype/index.cc

@@ -183,8 +183,7 @@ Error Index::Flush() {
   string unhidden = UnhiddenFile(dirname_, micros_);
   LOG(INFO) << "Wrote all index files for " << filename << ", moving to "
             << unhidden;
-  RETURN_IF_ERROR(Errno(0 == rename(filename.c_str(), unhidden.c_str())),
-                  "rename");
+  RETURN_IF_ERROR(Errno(rename(filename.c_str(), unhidden.c_str())), "rename");
   LOG(V1) << "Stored " << packets_ << " with " << ip4_.size() << " IP4 "
           << ip6_.size() << " IP6 " << proto_.size() << " protos "
           << port_.size() << " ports";

stenotype/make.sh

@@ -37,13 +37,14 @@ InstallPackage libleveldb-dev
 InstallPackage libsnappy-dev
 InstallPackage g++
 InstallPackage libcap2-bin
+InstallPackage libseccomp-dev
 
 echo "Building binary"
 g++ --std=c++0x \
   -o stenotype \
   -g -O3 -rdynamic -Wall \
   aio.cc util.cc packets.cc index.cc stenotype.cc \
-  -lleveldb -lrt -laio -lpthread -lsnappy
+  -lleveldb -lrt -laio -lpthread -lsnappy -lseccomp
 
 echo "Setting capabilities"
 chmod go-rwx stenotype

stenotype/packets.cc

@@ -41,6 +41,8 @@ inline size_t Align(size_t v) {
   return (v + TPACKET_ALIGNMENT - 1) & ((~TPACKET_ALIGNMENT) - 1);
 }
 
+const int kNoFanout = -1;
+
 }  // namespace
 
 namespace st {
@@ -165,8 +167,8 @@ PacketsV3::PacketsV3(PacketsV3::State* state) {
 Error PacketsV3::GetStats(Stats* stats) {
   struct tpacket_stats_v3 tpstats;
   socklen_t len = sizeof(tpstats);
-  RETURN_IF_ERROR(Errno(0 <= getsockopt(state_.fd, SOL_PACKET,
-                                        PACKET_STATISTICS, &tpstats, &len)),
+  RETURN_IF_ERROR(Errno(getsockopt(state_.fd, SOL_PACKET, PACKET_STATISTICS,
+                                   &tpstats, &len)),
                   "getsockopt PACKET_STATISTICS");
   stats_.drops += tpstats.tp_drops;
   *stats = stats_;
@@ -177,16 +179,23 @@ Error PacketsV3::Builder::Bind(const string& iface, PacketsV3** out) {
   RETURN_IF_ERROR(BadState(), "Builder");
 
   unsigned int ifindex = if_nametoindex(iface.c_str());
-  RETURN_IF_ERROR(Errno(ifindex != 0), "if_nametoindex");
+  if (ifindex == 0) {
+    return Errno();
+  }
   struct sockaddr_ll ll;
   memset(&ll, 0, sizeof(ll));
   ll.sll_family = AF_PACKET;
   ll.sll_protocol = htons(ETH_P_ALL);
   ll.sll_ifindex = ifindex;
   RETURN_IF_ERROR(
-      Errno(0 <= ::bind(state_.fd, reinterpret_cast<struct sockaddr*>(&ll),
-                        sizeof(ll))),
+      Errno(::bind(state_.fd, reinterpret_cast<struct sockaddr*>(&ll),
+                   sizeof(ll))),
       "bind");
+  if (fanout_ != kNoFanout) {
+    RETURN_IF_ERROR(Errno(setsockopt(state_.fd, SOL_PACKET, PACKET_FANOUT,
+                                     &fanout_, sizeof(fanout_))),
+                    "setting fanout");
+  }
   *out = new PacketsV3(&state_);
   return SUCCESS;
 }
@@ -196,35 +205,36 @@ Error PacketsV3::Builder::SetFilter(const string& filter) {
 
   int filter_size = filter.size();
   int filter_element_size = 4 + 2 + 2 + 8;
-  RETURN_IF_ERROR(Errno(0 == filter_size % filter_element_size),
-                  "invalid filter length");
+  if (filter_size % filter_element_size) {
+    return ERROR("invalid filter length");
+  }
   int num_structs = filter_size / filter_element_size;
-  RETURN_IF_ERROR(Errno(USHRT_MAX >= num_structs), "invalid filter: too long");
+  if (USHRT_MAX < num_structs) {
+    return ERROR("invalid filter: too long");
+  }
   struct sock_filter bpf_filter[num_structs];
   const char* data = filter.c_str();
   for (int i = 0; i < num_structs; i++) {
-    RETURN_IF_ERROR(Errno(4 == sscanf(data, "%4hx%2hhx%2hhx%8x",
-                                      &bpf_filter[i].code, &bpf_filter[i].jt,
-                                      &bpf_filter[i].jf, &bpf_filter[i].k)),
-                    "invalid filter");
+    if (4 != sscanf(data, "%4hx%2hhx%2hhx%8x", &bpf_filter[i].code,
+                    &bpf_filter[i].jt, &bpf_filter[i].jf, &bpf_filter[i].k)) {
+      return ERROR("invalid filter");
+    }
     data += filter_element_size;
   }
-  RETURN_IF_ERROR(Errno(0 == errno), "failure while parsing filter");
-
   struct sock_fprog bpf = {(unsigned short int)num_structs, bpf_filter};
-  RETURN_IF_ERROR(Errno(0 == setsockopt(state_.fd, SOL_SOCKET, SO_ATTACH_FILTER,
-                                        &bpf, sizeof(bpf))),
+  RETURN_IF_ERROR(Errno(setsockopt(state_.fd, SOL_SOCKET, SO_ATTACH_FILTER,
+                                   &bpf, sizeof(bpf))),
                   "so_attach_filter");
 #ifdef SO_LOCK_FILTER
   int v = 1;
   // SO_LOCK_FILTER is available only on kernels >= 3.9, so ignore the
   // ENOPROTOOPT
   // error here. We use it to make sure that no one can mess with our socket's
   // filter, so not having it is not really a big concern.
-  RETURN_IF_ERROR(Errno(0 == setsockopt(state_.fd, SOL_SOCKET, SO_LOCK_FILTER,
-                                        &v, sizeof(v)) ||
-                        errno == ENOPROTOOPT),
-                  "so_lock_filter");
+  RETURN_IF_ERROR(
+      Errno(setsockopt(state_.fd, SOL_SOCKET, SO_LOCK_FILTER, &v, sizeof(v)) ||
+            errno == ENOPROTOOPT),
+      "so_lock_filter");
   errno = 0;
 #endif
   return SUCCESS;
@@ -240,42 +250,39 @@ Error PacketsV3::Builder::BadState() {
 
 Error PacketsV3::Builder::SetVersion() {
   int version = TPACKET_V3;
-  return Errno(0 <= setsockopt(state_.fd, SOL_PACKET, PACKET_VERSION, &version,
-                               sizeof(version)));
+  return Errno(setsockopt(state_.fd, SOL_PACKET, PACKET_VERSION, &version,
+                          sizeof(version)));
 }
 
 Error PacketsV3::Builder::SetFanout(uint16_t fanout_type, uint16_t fanout_id) {
   RETURN_IF_ERROR(BadState(), "Builder");
+  // We can't actually set fanout until we bind, so just save it instead.
   LOG(V1) << "Setting fanout to type " << fanout_type << " ID " << fanout_id;
-  uint32_t fanout = fanout_type;
-  fanout <<= 16;
-  fanout |= fanout_id;
-  RETURN_IF_ERROR(Errno(0 <= setsockopt(state_.fd, SOL_PACKET, PACKET_FANOUT,
-                                        &fanout, sizeof(fanout))),
-                  "setting fanout options");
+  fanout_ = fanout_type;
+  fanout_ <<= 16;
+  fanout_ |= fanout_id;
   return SUCCESS;
 }
 
 Error PacketsV3::Builder::SetRingOptions(void* options, socklen_t size) {
-  RETURN_IF_ERROR(Errno(0 <= setsockopt(state_.fd, SOL_PACKET, PACKET_RX_RING,
-                                        options, size)),
-                  "setting socket ring options");
-  return SUCCESS;
+  return Errno(
+      setsockopt(state_.fd, SOL_PACKET, PACKET_RX_RING, options, size));
 }
 
 Error PacketsV3::Builder::MMapRing() {
   state_.ring = reinterpret_cast<char*>(
       mmap(NULL, state_.block_size * state_.num_blocks, PROT_READ | PROT_WRITE,
            MAP_SHARED | MAP_LOCKED | MAP_NORESERVE, state_.fd, 0));
-  RETURN_IF_ERROR(Errno(errno == 0), "mmap-ing ring");
+  if (state_.ring == MAP_FAILED) {
+    return Errno();
+  }
   return SUCCESS;
 }
 
 // socktype is SOCK_RAW or SOCK_DGRAM.
 Error PacketsV3::Builder::CreateSocket(int socktype) {
   state_.fd = socket(AF_PACKET, socktype, 0);
-  RETURN_IF_ERROR(Errno(state_.fd >= 0), "creating socket");
-  return SUCCESS;
+  return Errno(state_.fd);
 }
 
 Error PacketsV3::PollForPacket() {
@@ -285,9 +292,10 @@ Error PacketsV3::PollForPacket() {
   pfd.revents = 0;
   int64_t duration_micros = -GetCurrentTimeMicros();
   int ret = poll(&pfd, 1, -1);
+  Error out = Errno(ret);
   duration_micros += GetCurrentTimeMicros();
   SleepForMicroseconds(kMinPollMillis * kNumMicrosPerMilli - duration_micros);
-  return Errno(ret >= 0);
+  return out;
 }
 
 void PacketsV3::State::Swap(PacketsV3::State* s) {
@@ -298,7 +306,7 @@ void PacketsV3::State::Swap(PacketsV3::State* s) {
 }
 
 PacketsV3::State::~State() {
-  if (ring) {
+  if (ring != NULL && ring != MAP_FAILED) {
     munmap(ring, block_size * num_blocks);
   }
   if (fd >= 0) {
@@ -315,7 +323,7 @@ PacketsV3::~PacketsV3() {
   delete[] block_mus_;
 }
 
-PacketsV3::Builder::Builder() {}
+PacketsV3::Builder::Builder() : fanout_(kNoFanout) {}
 
 Error PacketsV3::Builder::SetUp(int socktype, struct tpacket_req3 tp) {
   if (tp.tp_block_size % getpagesize() != 0) {

stenotype/packets.h

@@ -177,6 +177,7 @@ class PacketsV3 {
     // State contains the state the builder sets up.  This state will be passed
     // to the PacketsV3 object created by Bind.
     State state_;
+    int fanout_;
   };
 
  private:

stenotype/remove_old.sh

@@ -19,7 +19,8 @@ function PercentFull {
 }
 
 function OldestFile {
-  for filename in $(ls -tr | grep -v INDEX); do
+  for filename in $(ls -tr PKT$THREAD/); do
+    filename="PKT$THREAD/$filename"
     if [ -f "$filename" ]; then
       echo "$filename"
       return
@@ -28,10 +29,11 @@ function OldestFile {
 }
 
 cd "$1"
+THREAD=$2
 
 while [ $(PercentFull) -gt 90 ]; do
   filename="$(OldestFile)"
   rm -fv "${filename}"
-  rm -fv "INDEX/${filename}"
+  rm -fv "${filename//PKT/IDX}"
   sleep 1
 done