Skip to content

Commit

Permalink
Handling closing of conns on dispatch loop and tracking deleted conns…
Browse files Browse the repository at this point in the history 
… on stats
  • Loading branch information
@oxtoacart
oxtoacart committed Apr 20, 2020
1 parent 98412e3 commit 4bf9075
Show file tree
Hide file tree
Showing 9 changed files with 251 additions and 73 deletions.
6 changes: 5 additions & 1 deletion conntrack_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,11 @@ func (s *server) createConntrackEntry(upFT FiveTuple) error {
func (s *server) deleteConntrackEntry(upFT FiveTuple) {
flow := s.ctFlowFor(false, upFT)
if err := s.ctrack.Delete(flow); err != nil {
log.Errorf("Unable to delete conntrack entry for %v: %v", upFT, err)
if log.IsTraceEnabled() {
// The below error is expected if the remote end closed the connection already, because
// the OS automatically deletes the conntrack entry if it received an RST packet.
log.Errorf("Unable to delete conntrack entry for %v: %v", upFT, err)
}
}
}

Expand Down
15 changes: 7 additions & 8 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,16 @@ module github.com/getlantern/gonat
go 1.12

require (
github.com/aristanetworks/goarista v0.0.0-20190514202536-8f808a500156 // indirect
github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7
github.com/getlantern/fdcount v0.0.0-20170105153814-6a6cb5839bc5
github.com/getlantern/golog v0.0.0-20190809085441-26e09e6dd330
github.com/getlantern/fdcount v0.0.0-20190912142506-f89afd7367c4
github.com/getlantern/golog v0.0.0-20190830074920-4ef2e798c2d7
github.com/getlantern/gotun v0.0.0-20190809092752-6d35bb1397ee
github.com/getlantern/grtrack v0.0.0-20160824195228-cbf67d3fa0fd
github.com/getlantern/mtime v0.0.0-20170117193331-ba114e4a82b0 // indirect
github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f
github.com/getlantern/mtime v0.0.0-20200417132445-23682092d1f7 // indirect
github.com/getlantern/ops v0.0.0-20200403153110-8476b16edcd6
github.com/google/gopacket v1.1.17
github.com/mdlayher/netlink v0.0.0-20190514163018-336c8d74f4a0 // indirect
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c
github.com/stretchr/testify v1.3.0
github.com/ti-mo/conntrack v0.0.0-20190323132511-733fb77b6071
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
github.com/stretchr/testify v1.5.1
github.com/ti-mo/conntrack v0.3.0
)
70 changes: 50 additions & 20 deletions go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
github.com/aristanetworks/goarista v0.0.0-20190514202536-8f808a500156 h1:sdAZ4pJ5nD/EzLkcw4AonvhgrU1aBKxx6ga1b7Psr9o=
github.com/aristanetworks/goarista v0.0.0-20190514202536-8f808a500156/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 h1:NRUJuo3v3WGC/g5YiyF790gut6oQr5f3FBI88Wv0dx4=
github.com/getlantern/context v0.0.0-20190109183933-c447772a6520/go.mod h1:L+mq6/vvYHKjCX2oez0CgEAJmbq1fbb/oNJIWQkBybY=
github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7 h1:6uJ+sZ/e03gkbqZ0kUG6mfKoqDb4XMAzMIwlajq19So=
github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7/go.mod h1:l+xpFBrCtDLpK9qNjxs+cHU6+BAdlBaxHqikB6Lku3A=
github.com/getlantern/fdcount v0.0.0-20170105153814-6a6cb5839bc5 h1:8Q9iN/V24EG01IgXEKVScth/rTXpplBxCYio/yIKtUw=
github.com/getlantern/fdcount v0.0.0-20170105153814-6a6cb5839bc5/go.mod h1:XZwE+iIlAgr64OFbXKFNCllBwV4wEipPx8Hlo2gZdbM=
github.com/getlantern/fdcount v0.0.0-20190912142506-f89afd7367c4 h1:JdD4XSaT6/j6InM7MT1E4WRvzR8gurxfq53A3ML3B/Q=
github.com/getlantern/fdcount v0.0.0-20190912142506-f89afd7367c4/go.mod h1:XZwE+iIlAgr64OFbXKFNCllBwV4wEipPx8Hlo2gZdbM=
github.com/getlantern/golog v0.0.0-20190809085441-26e09e6dd330 h1:BQIvwKkAWNoyQFtjk89XRV+GK0fT7Zvl1oHrp9Zhfl0=
github.com/getlantern/golog v0.0.0-20190809085441-26e09e6dd330/go.mod h1:zx/1xUUeYPy3Pcmet8OSXLbF47l+3y6hIPpyLWoR9oc=
github.com/getlantern/golog v0.0.0-20190830074920-4ef2e798c2d7 h1:guBYzEaLz0Vfc/jv0czrr2z7qyzTOGC9hiQ0VC+hKjk=
github.com/getlantern/golog v0.0.0-20190830074920-4ef2e798c2d7/go.mod h1:zx/1xUUeYPy3Pcmet8OSXLbF47l+3y6hIPpyLWoR9oc=
github.com/getlantern/gotun v0.0.0-20190809092752-6d35bb1397ee h1:wB9pX2HWLXeVMtxS/mWyF1UJZghEL4+YSSIj1mcBJfI=
github.com/getlantern/gotun v0.0.0-20190809092752-6d35bb1397ee/go.mod h1:zvsZQrsl7Yrmi+ENk5WZFT7dQaYtihAcI0H/9+LacqQ=
github.com/getlantern/grtrack v0.0.0-20160824195228-cbf67d3fa0fd h1:GPrx88jy222gMuRHXxBSViT/3zdNO210nRAaXn+lL6s=
Expand All @@ -18,46 +20,74 @@ github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7 h1:micT5vkcr9tOVk1F
github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7/go.mod h1:dD3CgOrwlzca8ed61CsZouQS5h5jIzkK9ZWrTcf0s+o=
github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55 h1:XYzSdCbkzOC0FDNrgJqGRo8PCMFOBFL9py72DRs7bmc=
github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55/go.mod h1:6mmzY2kW1TOOrVy+r41Za2MxXM+hhqTtY3oBKd2AgFA=
github.com/getlantern/mtime v0.0.0-20170117193331-ba114e4a82b0 h1:1VNkP55LM/W2IwWN+qi+5X3gZcEQHfj8X9E+FNxVgM4=
github.com/getlantern/mtime v0.0.0-20170117193331-ba114e4a82b0/go.mod h1:u537FS7ld4Whf7h7/0ql/myAudWWBNgeRhgE9XXH4Pk=
github.com/getlantern/mtime v0.0.0-20200417132445-23682092d1f7 h1:03J6Cb42EG06lHgpOFGm5BOax4qFqlSbSeKO2RGrj2g=
github.com/getlantern/mtime v0.0.0-20200417132445-23682092d1f7/go.mod h1:GfzwugvtH7YcmNIrHHizeyImsgEdyL88YkdnK28B14c=
github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f h1:wrYrQttPS8FHIRSlsrcuKazukx/xqO/PpLZzZXsF+EA=
github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f/go.mod h1:D5ao98qkA6pxftxoqzibIBBrLSUli+kYnJqrgBf9cIA=
github.com/getlantern/ops v0.0.0-20200403153110-8476b16edcd6 h1:QthAQCekS1YOeYWSvoHI6ZatlG4B+GBDLxV/2ZkBsTA=
github.com/getlantern/ops v0.0.0-20200403153110-8476b16edcd6/go.mod h1:D5ao98qkA6pxftxoqzibIBBrLSUli+kYnJqrgBf9cIA=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY=
github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
github.com/mdlayher/netlink v0.0.0-20190313131330-258ea9dff42c/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA=
github.com/mdlayher/netlink v0.0.0-20190514163018-336c8d74f4a0 h1:93Nlj3OMYLLA6QA6baiIeWW8P8CTlwQFA4pAecq7f/8=
github.com/mdlayher/netlink v0.0.0-20190514163018-336c8d74f4a0/go.mod h1:gOrA34zDL0K3RsACQe54bDYLF/CeFspQ9m5DOycycQ8=
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4 h1:nwOc1YaOrYJ37sEBrtWZrdqzK22hiJs3GpDmP3sR2Yw=
github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA=
github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
github.com/mdlayher/netlink v1.1.0 h1:mpdLgm+brq10nI9zM1BpX1kpDbh3NLl3RSnVq6ZSkfg=
github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw=
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8=
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/ti-mo/conntrack v0.0.0-20190323132511-733fb77b6071 h1:NnXFjutDwCYD2AoSFSEuUKC1H0f0Y8jeYXapxZSlg84=
github.com/ti-mo/conntrack v0.0.0-20190323132511-733fb77b6071/go.mod h1:4S0aZBgVjqDnOB7vilkmLDNHivyRGwPbDfTak5XQ0no=
github.com/ti-mo/netfilter v0.2.0 h1:mMZ70vvHTlY9y8ElWflp5nVN5kkUDvm6D1JXRgartKI=
github.com/ti-mo/netfilter v0.2.0/go.mod h1:8GbBGsY/8fxtyIdfwy29JiluNcPK4K7wIT+x42ipqUU=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/ti-mo/conntrack v0.3.0 h1:572/72R9la2FVvO6CbsLiCmR48U3pgCvIlLKoUrExDU=
github.com/ti-mo/conntrack v0.3.0/go.mod h1:tPSYNx21TnjxGz99pLD/lAN4fuEViaJZz+pliMqnovk=
github.com/ti-mo/netfilter v0.3.1 h1:+ZTmeTx+64Jw2N/1gmqm42kruDWjQ90SMjWEB1e6VDs=
github.com/ti-mo/netfilter v0.3.1/go.mod h1:t/5HvCCHA1LAYj/AZF2fWcJ23BQTA7lzTPCuwwi7xQY=
github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4=
github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5 h1:6M3SDHlHHDCx2PcQw3S4KsR170vGqDhJDOmpVd4Hjak=
golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 h1:rM0ROo5vb9AdYJi1110yjWGMej9ITfKddS89P3Fkhug=
golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190516110030-61b9204099cb h1:k07iPOt0d6nEnwXF+kHB+iEg+WSuKe/SOQuFM2QoD+E=
golang.org/x/sys v0.0.0-20190516110030-61b9204099cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA=
golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
90 changes: 68 additions & 22 deletions gonat_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,13 @@ type server struct {
ctTimeout uint32
randomPortSequence []uint16
portIndexes map[uint8]map[Addr]int
numConns int
connsByDownFT map[FiveTuple]*conn
connsByUpFT map[FiveTuple]*conn
fromDownstream chan *IPPacket
toDownstream chan *IPPacket
fromUpstream chan *IPPacket
closingConns chan *conn
closedConns chan *conn
close chan interface{}
closed chan interface{}
Expand Down Expand Up @@ -72,6 +74,7 @@ func NewServer(downstream ReadWriter, opts *Opts) (Server, error) {
fromDownstream: make(chan *IPPacket, opts.BufferDepth),
toDownstream: make(chan *IPPacket, opts.BufferDepth),
fromUpstream: make(chan *IPPacket, opts.BufferDepth),
closingConns: make(chan *conn, opts.BufferDepth),
closedConns: make(chan *conn, opts.BufferDepth),
close: make(chan interface{}),
closed: make(chan interface{}),
Expand All @@ -80,6 +83,9 @@ func NewServer(downstream ReadWriter, opts *Opts) (Server, error) {
}

func (s *server) Serve() error {
s.opts.StatsTracker.serverStarted()
defer s.opts.StatsTracker.serverClosed()

var err error
s.tcpSocket, err = createSocket(FiveTuple{IPProto: syscall.IPPROTO_TCP, Src: Addr{s.opts.IFAddr, 0}})
if err != nil {
Expand Down Expand Up @@ -109,11 +115,6 @@ func (s *server) Serve() error {

func (s *server) dispatch() {
defer func() {
for _, c := range s.connsByDownFT {
c.Close()
s.deleteConn(c)
s.deleteConntrackEntry(c.upFT)
}
close(s.toDownstream)
s.tcpSocket.Close()
s.udpSocket.Close()
Expand All @@ -127,15 +128,52 @@ func (s *server) dispatch() {
for {
select {
case pkt := <-s.fromDownstream:
log.Tracef("Got packet from downstream: %v", pkt.FT())
s.onPacketFromDownstream(pkt)
case pkt := <-s.fromUpstream:
log.Tracef("Got packet from upstream: %v", pkt.FT())
s.onPacketFromUpstream(pkt)
case c := <-s.closingConns:
s.deleteAndCloseConn(c)
case c := <-s.closedConns:
s.deleteConntrackEntry(c.upFT)
s.finalizeConn(c)
case <-reapTicker.C:
s.reapIdleConns()
case <-s.close:
return
if s.numConns == 0 {
// All connections already closed
return
}

// Close all connections
for _, c := range s.connsByDownFT {
s.deleteAndCloseConn(c)
}

// Then enter a special dispatch loop that handles closed cleanup
closeTimeout := s.opts.IdleTimeout * 2
closeTimer := time.NewTimer(closeTimeout)
for {
select {
case c := <-s.closingConns:
c.s.deleteAndCloseConn(c)
case c := <-s.closedConns:
s.finalizeConn(c)
if s.numConns == 0 {
// There are no more connections to clean up, we're done dispatching!
return
}
case <-reapTicker.C:
s.reapIdleConns()
case <-closeTimer.C:
log.Errorf("Failed to close server within %v, forcibly cleaning up %d connections and stopping dispatch", closeTimeout, len(s.connsByDownFT))
for _, c := range s.connsByDownFT {
s.deleteAndCloseConn(c)
s.finalizeConn(c)
}
return
}
}
}
}
}
Expand All @@ -149,7 +187,7 @@ func (s *server) onPacketFromDownstream(pkt *IPPacket) {

if pkt.HasTCPFlag(TCPFlagRST) {
if c != nil {
c.Close()
s.deleteAndCloseConn(c)
}
return
}
Expand All @@ -167,10 +205,11 @@ func (s *server) onPacketFromDownstream(pkt *IPPacket) {
s.dropPacket(pkt)
return
}
s.numConns++
s.connsByDownFT[downFT] = c
s.connsByUpFT[upFT] = c
c.markActive()
s.opts.StatsTracker.addConn(pkt.IPProto)
s.opts.StatsTracker.openedConn(pkt.IPProto)
}
select {
case c.toUpstream <- pkt:
Expand Down Expand Up @@ -246,26 +285,32 @@ func (s *server) assignPort(downFT FiveTuple) (upFT FiveTuple, err error) {
}

func (s *server) reapIdleConns() {
var connsToClose []*conn
for _, c := range s.connsByDownFT {
if c.timeSinceLastActive() > s.opts.IdleTimeout {
connsToClose = append(connsToClose, c)
s.deleteConn(c)
s.deleteAndCloseConn(c)
}
}
if len(connsToClose) > 0 {
// close conns on a goroutine to avoid tying up main dispatch loop
ops.Go(func() {
for _, c := range connsToClose {
c.Close()
}
})
}

func (s *server) requestCloseConn(c *conn) {
s.closingConns <- c
}

func (s *server) deleteAndCloseConn(c *conn) {
// Only delete the conn from our tracking maps if we actually closed it. It's possible that
// an already closed connection will attempt to double close, in which case we don't want
// to delete what's in the map because at this point it may contain a new, live connection
// with the same five tuple.
if c.doClose() {
delete(s.connsByDownFT, c.downFT)
delete(s.connsByUpFT, c.upFT)
}
}

func (s *server) deleteConn(c *conn) {
delete(s.connsByDownFT, c.downFT)
delete(s.connsByUpFT, c.upFT)
func (s *server) finalizeConn(c *conn) {
s.deleteConntrackEntry(c.upFT)
s.opts.StatsTracker.closedConn(c.upFT.IPProto)
s.numConns--
}

// readFromDownstream reads all IP packets from downstream clients.
Expand Down Expand Up @@ -341,6 +386,7 @@ func (s *server) Close() error {
case <-s.close:
// already closed
default:
s.opts.StatsTracker.startClosingServer()
close(s.close)
}
<-s.closed
Expand Down
4 changes: 3 additions & 1 deletion gonat_linux_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ const (
func TestEndToEnd(t *testing.T) {
RunTest(t, "tun0", "10.0.0.10", tunGW, "255.255.255.0", 1500, func(ifAddr string, dev io.ReadWriter, origEchoAddr Addr, finishedCh chan interface{}) (func() error, error) {
s, err := NewServer(&ReadWriterAdapter{dev}, &Opts{
IdleTimeout: 2 * time.Second,
StatsInterval: 250 * time.Millisecond,
BufferDepth: 1,
OnOutbound: func(pkt *IPPacket) {
pkt.SetDest(origEchoAddr)
},
Expand All @@ -32,7 +34,7 @@ func TestEndToEnd(t *testing.T) {
}

go func() {
assert.Equal(t, io.EOF, s.Serve())
s.Serve()
_s := s.(*server)
assert.True(t, _s.bufferPool.NumPooled() > 0, "buffers should be returned to pool")
_s.opts.StatsTracker.Close()
Expand Down

0 comments on commit 4bf9075

Please sign in to comment.