From b7713eb00c0abb76210dbe75abae40718d0748cc Mon Sep 17 00:00:00 2001 From: Ox Cart Date: Fri, 17 Apr 2020 22:02:17 -0500 Subject: [PATCH] Handling closing of conns on dispatch loop and tracking deleted conns on stats --- go.mod | 15 ++++--- go.sum | 70 ++++++++++++++++++++++--------- gonat_linux.go | 68 ++++++++++++++++++++++++++---- gonat_linux_test.go | 3 +- stats.go | 98 +++++++++++++++++++++++++++++++++++++++----- test_helper_linux.go | 10 ++++- transport_linux.go | 17 +++++++- tun_linux.go | 29 +++++++++++++ 8 files changed, 258 insertions(+), 52 deletions(-) create mode 100644 tun_linux.go diff --git a/go.mod b/go.mod index 58490e3..13f8d84 100644 --- a/go.mod +++ b/go.mod @@ -3,17 +3,16 @@ module github.com/getlantern/gonat go 1.12 require ( - github.com/aristanetworks/goarista v0.0.0-20190514202536-8f808a500156 // indirect github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7 - github.com/getlantern/fdcount v0.0.0-20170105153814-6a6cb5839bc5 - github.com/getlantern/golog v0.0.0-20190809085441-26e09e6dd330 + github.com/getlantern/fdcount v0.0.0-20190912142506-f89afd7367c4 + github.com/getlantern/golog v0.0.0-20190830074920-4ef2e798c2d7 github.com/getlantern/gotun v0.0.0-20190809092752-6d35bb1397ee github.com/getlantern/grtrack v0.0.0-20160824195228-cbf67d3fa0fd - github.com/getlantern/mtime v0.0.0-20170117193331-ba114e4a82b0 // indirect - github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f + github.com/getlantern/mtime v0.0.0-20200417132445-23682092d1f7 // indirect + github.com/getlantern/ops v0.0.0-20200403153110-8476b16edcd6 github.com/google/gopacket v1.1.17 - github.com/mdlayher/netlink v0.0.0-20190514163018-336c8d74f4a0 // indirect github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c - github.com/stretchr/testify v1.3.0 - github.com/ti-mo/conntrack v0.0.0-20190323132511-733fb77b6071 + github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 + github.com/stretchr/testify v1.5.1 + github.com/ti-mo/conntrack v0.3.0 ) diff --git a/go.sum b/go.sum index 5227338..9309561 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,17 @@ -github.com/aristanetworks/goarista v0.0.0-20190514202536-8f808a500156 h1:sdAZ4pJ5nD/EzLkcw4AonvhgrU1aBKxx6ga1b7Psr9o= -github.com/aristanetworks/goarista v0.0.0-20190514202536-8f808a500156/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 h1:NRUJuo3v3WGC/g5YiyF790gut6oQr5f3FBI88Wv0dx4= github.com/getlantern/context v0.0.0-20190109183933-c447772a6520/go.mod h1:L+mq6/vvYHKjCX2oez0CgEAJmbq1fbb/oNJIWQkBybY= github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7 h1:6uJ+sZ/e03gkbqZ0kUG6mfKoqDb4XMAzMIwlajq19So= github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7/go.mod h1:l+xpFBrCtDLpK9qNjxs+cHU6+BAdlBaxHqikB6Lku3A= -github.com/getlantern/fdcount v0.0.0-20170105153814-6a6cb5839bc5 h1:8Q9iN/V24EG01IgXEKVScth/rTXpplBxCYio/yIKtUw= -github.com/getlantern/fdcount v0.0.0-20170105153814-6a6cb5839bc5/go.mod h1:XZwE+iIlAgr64OFbXKFNCllBwV4wEipPx8Hlo2gZdbM= +github.com/getlantern/fdcount v0.0.0-20190912142506-f89afd7367c4 h1:JdD4XSaT6/j6InM7MT1E4WRvzR8gurxfq53A3ML3B/Q= +github.com/getlantern/fdcount v0.0.0-20190912142506-f89afd7367c4/go.mod h1:XZwE+iIlAgr64OFbXKFNCllBwV4wEipPx8Hlo2gZdbM= github.com/getlantern/golog v0.0.0-20190809085441-26e09e6dd330 h1:BQIvwKkAWNoyQFtjk89XRV+GK0fT7Zvl1oHrp9Zhfl0= github.com/getlantern/golog v0.0.0-20190809085441-26e09e6dd330/go.mod h1:zx/1xUUeYPy3Pcmet8OSXLbF47l+3y6hIPpyLWoR9oc= +github.com/getlantern/golog v0.0.0-20190830074920-4ef2e798c2d7 h1:guBYzEaLz0Vfc/jv0czrr2z7qyzTOGC9hiQ0VC+hKjk= +github.com/getlantern/golog v0.0.0-20190830074920-4ef2e798c2d7/go.mod h1:zx/1xUUeYPy3Pcmet8OSXLbF47l+3y6hIPpyLWoR9oc= github.com/getlantern/gotun v0.0.0-20190809092752-6d35bb1397ee h1:wB9pX2HWLXeVMtxS/mWyF1UJZghEL4+YSSIj1mcBJfI= github.com/getlantern/gotun v0.0.0-20190809092752-6d35bb1397ee/go.mod h1:zvsZQrsl7Yrmi+ENk5WZFT7dQaYtihAcI0H/9+LacqQ= github.com/getlantern/grtrack v0.0.0-20160824195228-cbf67d3fa0fd h1:GPrx88jy222gMuRHXxBSViT/3zdNO210nRAaXn+lL6s= @@ -18,46 +20,74 @@ github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7 h1:micT5vkcr9tOVk1F github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7/go.mod h1:dD3CgOrwlzca8ed61CsZouQS5h5jIzkK9ZWrTcf0s+o= github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55 h1:XYzSdCbkzOC0FDNrgJqGRo8PCMFOBFL9py72DRs7bmc= github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55/go.mod h1:6mmzY2kW1TOOrVy+r41Za2MxXM+hhqTtY3oBKd2AgFA= -github.com/getlantern/mtime v0.0.0-20170117193331-ba114e4a82b0 h1:1VNkP55LM/W2IwWN+qi+5X3gZcEQHfj8X9E+FNxVgM4= -github.com/getlantern/mtime v0.0.0-20170117193331-ba114e4a82b0/go.mod h1:u537FS7ld4Whf7h7/0ql/myAudWWBNgeRhgE9XXH4Pk= +github.com/getlantern/mtime v0.0.0-20200417132445-23682092d1f7 h1:03J6Cb42EG06lHgpOFGm5BOax4qFqlSbSeKO2RGrj2g= +github.com/getlantern/mtime v0.0.0-20200417132445-23682092d1f7/go.mod h1:GfzwugvtH7YcmNIrHHizeyImsgEdyL88YkdnK28B14c= github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f h1:wrYrQttPS8FHIRSlsrcuKazukx/xqO/PpLZzZXsF+EA= github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f/go.mod h1:D5ao98qkA6pxftxoqzibIBBrLSUli+kYnJqrgBf9cIA= +github.com/getlantern/ops v0.0.0-20200403153110-8476b16edcd6 h1:QthAQCekS1YOeYWSvoHI6ZatlG4B+GBDLxV/2ZkBsTA= +github.com/getlantern/ops v0.0.0-20200403153110-8476b16edcd6/go.mod h1:D5ao98qkA6pxftxoqzibIBBrLSUli+kYnJqrgBf9cIA= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= -github.com/mdlayher/netlink v0.0.0-20190313131330-258ea9dff42c/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= -github.com/mdlayher/netlink v0.0.0-20190514163018-336c8d74f4a0 h1:93Nlj3OMYLLA6QA6baiIeWW8P8CTlwQFA4pAecq7f/8= -github.com/mdlayher/netlink v0.0.0-20190514163018-336c8d74f4a0/go.mod h1:gOrA34zDL0K3RsACQe54bDYLF/CeFspQ9m5DOycycQ8= +github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= +github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4 h1:nwOc1YaOrYJ37sEBrtWZrdqzK22hiJs3GpDmP3sR2Yw= +github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= +github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= +github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= +github.com/mdlayher/netlink v1.1.0 h1:mpdLgm+brq10nI9zM1BpX1kpDbh3NLl3RSnVq6ZSkfg= +github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/ti-mo/conntrack v0.0.0-20190323132511-733fb77b6071 h1:NnXFjutDwCYD2AoSFSEuUKC1H0f0Y8jeYXapxZSlg84= -github.com/ti-mo/conntrack v0.0.0-20190323132511-733fb77b6071/go.mod h1:4S0aZBgVjqDnOB7vilkmLDNHivyRGwPbDfTak5XQ0no= -github.com/ti-mo/netfilter v0.2.0 h1:mMZ70vvHTlY9y8ElWflp5nVN5kkUDvm6D1JXRgartKI= -github.com/ti-mo/netfilter v0.2.0/go.mod h1:8GbBGsY/8fxtyIdfwy29JiluNcPK4K7wIT+x42ipqUU= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/ti-mo/conntrack v0.3.0 h1:572/72R9la2FVvO6CbsLiCmR48U3pgCvIlLKoUrExDU= +github.com/ti-mo/conntrack v0.3.0/go.mod h1:tPSYNx21TnjxGz99pLD/lAN4fuEViaJZz+pliMqnovk= +github.com/ti-mo/netfilter v0.3.1 h1:+ZTmeTx+64Jw2N/1gmqm42kruDWjQ90SMjWEB1e6VDs= +github.com/ti-mo/netfilter v0.3.1/go.mod h1:t/5HvCCHA1LAYj/AZF2fWcJ23BQTA7lzTPCuwwi7xQY= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5 h1:6M3SDHlHHDCx2PcQw3S4KsR170vGqDhJDOmpVd4Hjak= -golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 h1:rM0ROo5vb9AdYJi1110yjWGMej9ITfKddS89P3Fkhug= -golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190516110030-61b9204099cb h1:k07iPOt0d6nEnwXF+kHB+iEg+WSuKe/SOQuFM2QoD+E= golang.org/x/sys v0.0.0-20190516110030-61b9204099cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/gonat_linux.go b/gonat_linux.go index 04db76f..79880f5 100644 --- a/gonat_linux.go +++ b/gonat_linux.go @@ -28,6 +28,7 @@ type server struct { fromDownstream chan *IPPacket toDownstream chan *IPPacket fromUpstream chan *IPPacket + closingConns chan *conn closedConns chan *conn close chan interface{} closed chan interface{} @@ -72,6 +73,7 @@ func NewServer(downstream ReadWriter, opts *Opts) (Server, error) { fromDownstream: make(chan *IPPacket, opts.BufferDepth), toDownstream: make(chan *IPPacket, opts.BufferDepth), fromUpstream: make(chan *IPPacket, opts.BufferDepth), + closingConns: make(chan *conn, opts.BufferDepth), closedConns: make(chan *conn, opts.BufferDepth), close: make(chan interface{}), closed: make(chan interface{}), @@ -80,6 +82,9 @@ func NewServer(downstream ReadWriter, opts *Opts) (Server, error) { } func (s *server) Serve() error { + s.opts.StatsTracker.addServer() + defer s.opts.StatsTracker.removeServer() + var err error s.tcpSocket, err = createSocket(FiveTuple{IPProto: syscall.IPPROTO_TCP, Src: Addr{s.opts.IFAddr, 0}}) if err != nil { @@ -109,11 +114,6 @@ func (s *server) Serve() error { func (s *server) dispatch() { defer func() { - for _, c := range s.connsByDownFT { - c.Close() - s.deleteConn(c) - s.deleteConntrackEntry(c.upFT) - } close(s.toDownstream) s.tcpSocket.Close() s.udpSocket.Close() @@ -127,15 +127,52 @@ func (s *server) dispatch() { for { select { case pkt := <-s.fromDownstream: + log.Tracef("Got packet from downstream: %v", pkt.FT()) s.onPacketFromDownstream(pkt) case pkt := <-s.fromUpstream: + log.Tracef("Got packet from upstream: %v", pkt.FT()) s.onPacketFromUpstream(pkt) + case c := <-s.closingConns: + s.deleteAndCloseConn(c) case c := <-s.closedConns: - s.deleteConntrackEntry(c.upFT) + s.finalizeConn(c) case <-reapTicker.C: s.reapIdleConns() case <-s.close: - return + if len(s.connsByDownFT) == 0 { + // All connections already closed + return + } + + // Close all connections + for _, c := range s.connsByDownFT { + c.Close() + } + + // Then enter a special dispatch loop that handles closed cleanup + closeTimeout := s.opts.IdleTimeout * 2 + closeTimer := time.NewTimer(closeTimeout) + for { + select { + case c := <-s.closingConns: + c.s.deleteAndCloseConn(c) + case c := <-s.closedConns: + s.finalizeConn(c) + if len(s.connsByDownFT) == 0 { + // There are no more connections to cleanup, we're done dispatching! + return + } + case <-reapTicker.C: + s.reapIdleConns() + case <-closeTimer.C: + log.Errorf("Failed to close server within %v, forcibly cleaning up %d connections and stopping dispatch", closeTimeout, len(s.connsByDownFT)) + for _, c := range s.connsByDownFT { + s.deleteAndCloseConn(c) + s.finalizeConn(c) + } + return + } + } } } } @@ -250,7 +287,6 @@ func (s *server) reapIdleConns() { for _, c := range s.connsByDownFT { if c.timeSinceLastActive() > s.opts.IdleTimeout { connsToClose = append(connsToClose, c) - s.deleteConn(c) } } if len(connsToClose) > 0 { @@ -263,9 +299,22 @@ func (s *server) reapIdleConns() { } } -func (s *server) deleteConn(c *conn) { +func (s *server) requestCloseConn(c *conn) { + s.closingConns <- c + s.opts.StatsTracker.addClosingConn(c.upFT.IPProto) +} + +func (s *server) deleteAndCloseConn(c *conn) { delete(s.connsByDownFT, c.downFT) delete(s.connsByUpFT, c.upFT) + c.doClose() +} + +func (s *server) finalizeConn(c *conn) { + s.deleteConntrackEntry(c.upFT) + s.opts.StatsTracker.removeConn(c.upFT.IPProto) + s.opts.StatsTracker.addClosedConn(c.upFT.IPProto) + } // readFromDownstream reads all IP packets from downstream clients. @@ -341,6 +390,7 @@ func (s *server) Close() error { case <-s.close: // already closed default: + s.opts.StatsTracker.startClosingServer() close(s.close) } <-s.closed diff --git a/gonat_linux_test.go b/gonat_linux_test.go index 4a168b2..5095ba0 100644 --- a/gonat_linux_test.go +++ b/gonat_linux_test.go @@ -19,6 +19,7 @@ const ( func TestEndToEnd(t *testing.T) { RunTest(t, "tun0", "10.0.0.10", tunGW, "255.255.255.0", 1500, func(ifAddr string, dev io.ReadWriter, origEchoAddr Addr, finishedCh chan interface{}) (func() error, error) { s, err := NewServer(&ReadWriterAdapter{dev}, &Opts{ + IdleTimeout: 2 * time.Second, StatsInterval: 250 * time.Millisecond, OnOutbound: func(pkt *IPPacket) { pkt.SetDest(origEchoAddr) @@ -32,7 +33,7 @@ func TestEndToEnd(t *testing.T) { } go func() { - assert.Equal(t, io.EOF, s.Serve()) + s.Serve() _s := s.(*server) assert.True(t, _s.bufferPool.NumPooled() > 0, "buffers should be returned to pool") _s.opts.StatsTracker.Close() diff --git a/stats.go b/stats.go index 81898ea..6ccec6e 100644 --- a/stats.go +++ b/stats.go @@ -11,15 +11,22 @@ import ( // StatsTracker tracks statistics for one or more gonat servers. type StatsTracker struct { - acceptedPackets int64 - invalidPackets int64 - droppedPackets int64 - numTCPConns int64 - numUDPConns int64 - statsInterval time.Duration - startOnce sync.Once - stop chan interface{} - stopped chan interface{} + acceptedPackets int64 + invalidPackets int64 + droppedPackets int64 + numServers int64 + numServersClosing int64 + numServersClosed int64 + numTCPConns int64 + numUDPConns int64 + numTCPConnsClosing int64 + numUDPConnsClosing int64 + numTCPConnsClosed int64 + numUDPConnsClosed int64 + statsInterval time.Duration + startOnce sync.Once + stop chan interface{} + stopped chan interface{} } // NewStatsTracker creates a new StatsTracker that will log stats at the given statsInterval. @@ -61,7 +68,10 @@ func (s *StatsTracker) trackStats() { case <-s.stop: return case <-ticker.C: - log.Debugf("TCP Conns: %v UDP Conns: %v", s.NumTCPConns(), s.NumUDPConns()) + log.Debugf("Servers: %d Closing: %d Closed: %d", s.NumServers(), s.NumServersClosing(), s.NumServersClosed()) + log.Debugf("TCP Conns: %d UDP Conns: %d", s.NumTCPConns(), s.NumUDPConns()) + log.Debugf("TCP Conns Closing: %d TCP Conns Closed: %d", s.NumTCPConnsClosing(), s.NumTCPConnsClosed()) + log.Debugf("UDP Conns Closing: %d UDP Conns Closed: %d", s.NumUDPConnsClosing(), s.NumUDPConnsClosed()) log.Debugf("Invalid Packets: %d Accepted Packets: %d Dropped Packets: %d", s.InvalidPackets(), s.AcceptedPackets(), s.DroppedPackets()) } } @@ -95,6 +105,34 @@ func (s *StatsTracker) DroppedPackets() int { return int(atomic.LoadInt64(&s.droppedPackets)) } +func (s *StatsTracker) addServer() { + atomic.AddInt64(&s.numServers, 1) +} + +func (s *StatsTracker) startClosingServer() { + atomic.AddInt64(&s.numServersClosing, 1) +} + +func (s *StatsTracker) removeServer() { + atomic.AddInt64(&s.numServersClosing, -1) + atomic.AddInt64(&s.numServers, -1) +} + +// NumServers gives a count of the number of gonat servers currently running +func (s *StatsTracker) NumServers() int { + return int(atomic.LoadInt64(&s.numServers)) +} + +// NumServersClosing gives a count of the number of gonat servers currently closing +func (s *StatsTracker) NumServersClosing() int { + return int(atomic.LoadInt64(&s.numServersClosing)) +} + +// NumServersClosed gives a count of the number of gonat servers closed +func (s *StatsTracker) NumServersClosed() int { + return int(atomic.LoadInt64(&s.numServersClosed)) +} + func (s *StatsTracker) addConn(proto uint8) { switch proto { case syscall.IPPROTO_TCP: @@ -122,3 +160,43 @@ func (s *StatsTracker) NumTCPConns() int { func (s *StatsTracker) NumUDPConns() int { return int(atomic.LoadInt64(&s.numUDPConns)) } + +func (s *StatsTracker) addClosingConn(proto uint8) { + switch proto { + case syscall.IPPROTO_TCP: + atomic.AddInt64(&s.numTCPConnsClosing, 1) + case syscall.IPPROTO_UDP: + atomic.AddInt64(&s.numUDPConnsClosing, 1) + } +} + +func (s *StatsTracker) addClosedConn(proto uint8) { + switch proto { + case syscall.IPPROTO_TCP: + atomic.AddInt64(&s.numTCPConnsClosing, -1) + atomic.AddInt64(&s.numTCPConnsClosed, 1) + case syscall.IPPROTO_UDP: + atomic.AddInt64(&s.numUDPConnsClosing, -1) + atomic.AddInt64(&s.numUDPConnsClosed, 1) + } +} + +// NumTCPConnsClosing gives a count of the number of TCP connections being closed +func (s *StatsTracker) NumTCPConnsClosing() int { + return int(atomic.LoadInt64(&s.numTCPConnsClosing)) +} + +// NumTCPConnsClosed gives a count of the number of TCP connections that have been closed +func (s *StatsTracker) NumTCPConnsClosed() int { + return int(atomic.LoadInt64(&s.numTCPConnsClosed)) +} + +// NumUDPConnsClosing gives a count of the number of UDP connections being closed +func (s *StatsTracker) NumUDPConnsClosing() int { + return int(atomic.LoadInt64(&s.numUDPConnsClosing)) +} + +// NumUDPConnsClosed gives a count of the number of UDP connections that have been closed +func (s *StatsTracker) NumUDPConnsClosed() int { + return int(atomic.LoadInt64(&s.numUDPConnsClosed)) +} diff --git a/test_helper_linux.go b/test_helper_linux.go index fda1dd2..e6e0266 100644 --- a/test_helper_linux.go +++ b/test_helper_linux.go @@ -10,7 +10,6 @@ import ( "time" "github.com/getlantern/fdcount" - tun "github.com/getlantern/gotun" "github.com/getlantern/grtrack" "github.com/stretchr/testify/assert" @@ -41,7 +40,8 @@ func RunTest(t *testing.T, tunDeviceName, tunAddr, tunGW, tunMask string, mtu in } // Open a TUN device - dev, err := tun.OpenTunDevice(tunDeviceName, tunAddr, tunGW, tunMask, mtu) + log.Debugf("Opening TUN device at %v", tunAddr) + dev, err := TUNDevice(tunDeviceName, tunAddr, tunMask, mtu) if err != nil { if err != nil { if strings.HasSuffix(err.Error(), "operation not permitted") { @@ -83,16 +83,21 @@ func RunTest(t *testing.T, tunDeviceName, tunAddr, tunGW, tunMask string, mtu in return } + log.Debug("Writing to UDP") _, err = uconn.Write([]byte("helloudp")) if !assert.NoError(t, err) { return } + log.Debug("Reading from UDP") _, err = io.ReadFull(uconn, b) if !assert.NoError(t, err) { return } + + log.Debug("Closing udp conn") uconn.Close() + log.Debug("Closed udp conn") log.Debugf("Dialing echo server with TCP at: %v", echoAddr) conn, err := net.DialTimeout("tcp", echoAddr, 5*time.Second) @@ -147,6 +152,7 @@ func tcpEcho(t *testing.T, closeCh <-chan interface{}, ip string) string { go func() { for { conn, err := l.Accept() + log.Debugf("TCP echo accepted, error? %v", err) if err != nil { return } diff --git a/transport_linux.go b/transport_linux.go index 6b1c412..af9c231 100644 --- a/transport_linux.go +++ b/transport_linux.go @@ -94,6 +94,9 @@ func (c *conn) writeToUpstream() { c.s.bufferPool.PutSlice(pkt.Raw) if err != nil { log.Errorf("Error writing upstream: %v", err) + c.Close() + // Wait for conn to have actually been closed + <-c.close return } c.markActive() @@ -114,9 +117,19 @@ func (c *conn) timeSinceLastActive() time.Duration { func (c *conn) Close() error { select { case <-c.close: - return nil + // already closed + default: + // Ask server to close this conn so that it happens in the dispatch loop + c.s.requestCloseConn(c) + } + return nil +} + +func (c *conn) doClose() { + select { + case <-c.close: + // already closed default: close(c.close) - return nil } } diff --git a/tun_linux.go b/tun_linux.go new file mode 100644 index 0000000..69f500c --- /dev/null +++ b/tun_linux.go @@ -0,0 +1,29 @@ +package gonat + +import ( + "io" + "os/exec" + "strconv" + + "github.com/getlantern/errors" + "github.com/songgao/water" +) + +// TUNDevice creates a TUN device with the given name and configures an interface for that TUN +// device at the given address and netmask and given mtu (should usually be 1500). +func TUNDevice(name, addr, netmask string, mtu int) (io.ReadWriteCloser, error) { + cfg := water.Config{DeviceType: water.TUN} + cfg.Name = name + dev, err := water.New(cfg) + if err != nil { + return nil, errors.New("error opening TUN device: %v", err) + } + log.Debugf("Created TUN device named %v", dev.Name()) + + if out, configErr := exec.Command("ifconfig", dev.Name(), addr, "netmask", netmask, "mtu", strconv.Itoa(mtu)).CombinedOutput(); configErr != nil { + dev.Close() + return nil, errors.New("failed to configure tun device address: %v", string(out)) + } + + return dev, nil +}