Large PR - Adapter to C code, demo polling implementation

CMakeLists.txt

@@ -66,6 +66,7 @@ PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS
     ${SRC_DIR}/tcpinfo.proto)
 add_library(tcpinfo_lib
     ${SRC_DIR}/tcpinfo_lib.cc ${SRC_DIR}/tcpinfo_lib.h
+    ${SRC_DIR}/tcpinfo_c_adapter.h ${SRC_DIR}/tcpinfo_c_adapter.c
     ${PROTO_HDRS} ${PROTO_SRCS})
 add_dependencies(tcpinfo_lib iproute2)
 
@@ -89,6 +90,17 @@ target_link_libraries(tcpinfo_proto_test
 
 add_test(tcpinfo_proto tcpinfo_proto_test)
 
+add_executable(tcpinfo_lib_test
+    ${SRC_DIR}/tcpinfo_lib_test.cc)
+add_dependencies(tcpinfo_lib_test googletest)
+
+target_link_libraries(tcpinfo_lib_test
+    ${GTEST_LIBS_DIR}/libgtest.a
+    ${GTEST_LIBS_DIR}/libgtest_main.a
+    tcpinfo_lib)
+
+add_test(tcpinfo_lib tcpinfo_lib_test)
+
 ################### connection_cache library  ########################
 add_library(connection_cache_lib
     ${SRC_DIR}/connection_cache.cc ${SRC_DIR}/connection_cache.h)
@@ -107,3 +119,13 @@ target_link_libraries(connection_cache_test
 
 add_test(connection_cache connection_cache_test)
 
+################### tcpinfo polling proto ########################
+add_executable(poll
+    ${SRC_DIR}/main.cc
+    ${SRC_DIR}/tcpinfo_c_adapter.c)
+add_dependencies(poll iproute2 tcpinfo_lib)
+target_link_libraries(poll
+    ${IPROUTE2_LIBS_DIR}/libnetlink.a
+    ${IPROUTE2_LIBS_DIR}/libutil.a
+    connection_cache_lib
+    tcpinfo_lib)

README.md

@@ -11,24 +11,53 @@ sudo apt-get install build-essential autoconf automake make cmake
 sudo apt-get install clang g++ libtool curl unzip bison flex
 ```
 
+## clang
+The build is configured to use clang.  If you want to use a different compiler,
+export CXX_COMPILER and C_COMPILER.
+
 # Building
 ```
-git clone git@github.com:gfr10598/tcpinfo_lib
+git clone git@github.com:mlab/tcpinfo_lib
 cd tcpinfo_lib
 mkdir build
 cd build
 cmake .. && cmake --build . && ctest -V
 ```
+When make finishes, there should be a directory tree under .../build that
+contains all config, code, object files, libraries, and binaries.
 
-# Dependencies
-There are several dependencies that are handled through add_directory(...).  The
-CMake files for these are under ext/, and they are invoked as needed.
+# Travis-CI integration
+.travis.yml provides config for building within travis-ci.  It includes a gcc
+config that also produces coverage data for coveralls, and a clang config that
+builds optimized code.
 
-The install_protobuf.sh script is used for building and installing google protobuf as needed.
+## protobuf compiler
+CMake apparently uses pkg-config to discover the protobuf compiler.  But it
+appears that when installed with apt-get install protobuf-compiler, (at least
+on gobuntu) the pkg-config .pc file is not installed.
+So, to make everything work smoothly, cmake .. will run install-protobuf.sh,
+which will install it from github if pkg-config doesn't have a suitable config.
 
-# Travis-CI integration
-.travis.yml provides config for building within travis-ci.  Currently only gcc
-config is working, but will hopefully add clang config later.
+This requires sudo privileges, and you may be prompted for your password for
+the install and ldconfig steps.
+
+If you later want to revert your system, you will need to cd to the protobuf
+directory, and:
+```
+sudo make uninstall
+```
+To restore the original protobuf-compiler (if you had one), you may also need
+to:
+```
+sudo apt-get uninstall protobuf-compiler
+sudo apt-get install protobuf-compiler
+```
+
+# Dependency on iproute2 and gtest
+The ext/iproute2 and ext/gtest directories provide rules for downloading
+and building the dependencies.  They are incorporated with add_subdirectory
+and should be downloaded and built when needed.  They should show up under
+build/ext/...
 
 # Source tree
 ```
@@ -38,16 +67,22 @@ config is working, but will hopefully add clang config later.
 │   ├── gtest
 │   │   └── CMakeLists.txt
 │   └── iproute2
-│       └── CMakeLists.txt
+│       └── CMakeLists.txt
 ├── install-protobuf.sh
 ├── LICENSE
+├── pre-commit
 ├── README.md
 └── src
     ├── connection_cache.cc
     ├── connection_cache.h
     ├── connection_cache_test.cc
+    ├── main.cc
+    ├── README.md
+    ├── tcpinfo_c_adapter.c
+    ├── tcpinfo_c_adapter.h
     ├── tcpinfo_lib.cc
     ├── tcpinfo_lib.h
+    ├── tcpinfo_lib_test.cc
     ├── tcpinfo.proto
     └── tcpinfo_proto_test.cc
 ```

src/README.md

@@ -0,0 +1,16 @@
+# Components
+## tcpinfo.proto
+Data structures for all tcp_info and netlink related code.
+
+## tcpinfo_lib
+Utility library providing high level api to netlink services.
+
+## connection_cache
+Component for caching data for each active (non-local) connection.
+
+## tcpinfo_c_adapter
+Low level code that makes calls to the netlink library and iproute2 code.
+
+## main.cc
+Simple demo polling program illustrating use of the tcpinfo_lib library.
+

src/connection_cache.cc

@@ -21,7 +21,7 @@
 
 extern "C" {
 #include <linux/inet_diag.h>
-#include <sys/socket.h>
+#include <sys/socket.h>  // For AF_INET
 }
 
 namespace mlab {

src/main.cc

@@ -0,0 +1,108 @@
+/**********************************************************************
+ * Demo polling implementation.
+ *
+ * This captures 10 minutes of connection data.
+ * Raw nlmsg data compressed with bzip2 takes about 15 bytes/record.
+ * Wire format protos compressed with bzip2 also takes about 15 bytes/record.
+ * bzip2 compressed proto.ShortDebugString takes about 16 bytes/record.
+ **********************************************************************/
+
+#include <chrono>
+#include <fstream>
+#include <unistd.h>
+
+#include "tcpinfo_lib.h"  // Poller
+
+namespace {
+// Dump just raw nlmsg, without protocol info.
+void DumpNlMsg(const std::string& nlmsg) {
+  std::ofstream out;
+  out.open("nldata", std::ofstream::app | std::ofstream::binary);
+  int len = nlmsg.size();
+  for (int i = 0; i < len; ++i) {
+    out.put(nlmsg.at(i));
+  }
+  out.close();
+}
+
+void DumpProto(const mlab::netlink::TCPDiagnosticsProto& proto) {
+  std::ofstream out;
+  out.open("protodata", std::ofstream::app | std::ofstream::binary);
+  proto.SerializeToOstream(&out);
+  out.close();
+
+  // TODO(gfr) This should be optional.
+  printf("%s\n", proto.ShortDebugString().c_str());
+}
+
+void DumpSummary(const mlab::netlink::TCPDiagnosticsProto& proto, std::string tag) {
+  const google::protobuf::EnumDescriptor* enum_desc =
+          mlab::netlink::TCPState_descriptor();
+  fprintf(stderr, "%5d %s %s\n",
+          proto.inet_diag_msg().sock_id().source().port(),
+          tag.c_str(),
+          enum_desc->value(
+              proto.inet_diag_msg().state())
+                  ->options().GetExtension(mlab::netlink::name).c_str());
+}
+
+void Output(const std::string& nlmsg, int protocol, std::string tag) {
+  DumpNlMsg(nlmsg);
+
+  auto proto = mlab::netlink::TCPInfoParser().ParseNLMsg(
+      nlmsg, mlab::netlink::Protocol(protocol));
+
+  DumpProto(proto);
+  DumpSummary(proto, tag);
+}
+
+// Output, using '#' as tag for summary.
+void on_close(int protocol, const std::string& old_msg,
+              const std::string& new_msg) {
+  Output(old_msg, protocol, "#");
+}
+
+// Output each new state we see, except for ESTABLISHED, which should only
+// be output when it is the old state.
+// PREREQ: old_msg and new_msg should have different TCPState values.
+void on_new_state(int protocol, const std::string& old_msg,
+                  const std::string& new_msg) {
+  // Output old data when it's state is ESTABLISHED.
+  if (!old_msg.empty()) {
+    auto old_state = mlab::netlink::GetStateFromStr(old_msg);
+
+    if (old_state == mlab::netlink::TCPState::ESTABLISHED) {
+      Output(old_msg, protocol, "*");
+    }
+  }
+
+  // For all states EXCEPT ESTABLISHED, output the state immediately.
+  if (mlab::netlink::GetStateFromStr(new_msg)
+      != mlab::netlink::TCPState::ESTABLISHED) {
+    Output(new_msg, protocol, " ");
+  }
+}
+} // anonymous namespace
+
+extern mlab::netlink::TCPInfoPoller g_poller_;
+
+int main(int argc, char* argv[]) {
+  using std::chrono::steady_clock;
+  using std::chrono::duration;
+
+  auto start = steady_clock::now();
+  auto one_minute = duration<int>(60);
+
+  // NOTE: This fires only occasionally, on very short lived connections.
+  // Usually, we see some other OnNewState before the connection closes.
+  g_poller_.OnClose(on_close, {mlab::netlink::TCPState::ESTABLISHED});
+  g_poller_.OnNewState(on_new_state);
+
+  int count = 0;
+  while (steady_clock::now() < start + 10*one_minute) {
+    g_poller_.PollOnce();
+    count++;
+  }
+  fprintf(stderr, "Rate was %d polls/sec\n", count / 600);
+  return 0;
+}

src/tcpinfo.proto

@@ -82,10 +82,12 @@ enum TCPState {  //  from tcp_states.h
 
 message InetDiagMsgProto {
   enum AddressFamily {
+    // NOTE: these are equivalent to AF_... in socket.h, but cannot have the
+    // same names since those are macros and will cause collisions.
     // There are many other families, but for now we only care about these.
-    AF_UNSPEC = 0;
-    AF_INET = 2;
-    AF_INET6 = 10;
+    UNSPEC = 0;
+    INET = 2;
+    INET6 = 10;
   }
   // These are 8 bit unsigned.
   optional AddressFamily family = 1;