From bbad83048890c00e64df57ec9d97741de983fc76 Mon Sep 17 00:00:00 2001 From: Shrey Shah Date: Fri, 10 Nov 2023 12:09:58 +0530 Subject: [PATCH 1/7] [POA-155] Capture client/server timeouts as well as Agent parsing errors --- go.mod | 4 +-- go.sum | 34 ++++++++++++++++++--- learn/parse_http.go | 15 +++++++++ pcap/stream.go | 45 +++++++++++++++++++++++++++ trace/backend_collector.go | 62 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 153 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 11c06649..ee4309f9 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/AlecAivazis/survey/v2 v2.3.6 github.com/OneOfOne/xxhash v1.2.8 github.com/Pallinder/go-randomdata v1.2.0 - github.com/akitasoftware/akita-ir v0.0.0-20220630210013-8926783978fe - github.com/akitasoftware/akita-libs v0.0.0-20230708003852-6c8da9931921 + github.com/akitasoftware/akita-ir v0.0.0-20231103112405-e2221503d639 // todo: update this to match the commit on master once akita-ir#10 is merged + github.com/akitasoftware/akita-libs v0.0.0-20231110062924-14f503bf0c85 // todo: update this to match the commit on master once akita-libs#201 is merged github.com/akitasoftware/go-utils v0.0.0-20221207014235-6f4c9079488d github.com/akitasoftware/plugin-flickr v0.2.0 github.com/andybalholm/brotli v1.0.1 diff --git a/go.sum b/go.sum index 91906993..c294c93a 100644 --- a/go.sum +++ b/go.sum @@ -28,11 +28,11 @@ github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00 github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= github.com/akitasoftware/akita-cli v0.18.5/go.mod h1:xWKdto84IENKjzmS3ctCQWGetfYYhGDW3XNZY2/RduY= github.com/akitasoftware/akita-ir v0.0.0-20211020161529-944af4d11d6e/go.mod h1:WEWPzhZtxlJnov3MxcqSDiZaHHf00vs3aJwCdt3OwzA= -github.com/akitasoftware/akita-ir v0.0.0-20220630210013-8926783978fe h1:0BeBDjLDFPwv2bkk6YuRAPf1r6U4Wby98NHI9+Lddvs= -github.com/akitasoftware/akita-ir v0.0.0-20220630210013-8926783978fe/go.mod h1:WEWPzhZtxlJnov3MxcqSDiZaHHf00vs3aJwCdt3OwzA= +github.com/akitasoftware/akita-ir v0.0.0-20231103112405-e2221503d639 h1:hIWGdk/RJGbvfLA2pODQLn8lAygHXEQLZN3VuIrkCHU= +github.com/akitasoftware/akita-ir v0.0.0-20231103112405-e2221503d639/go.mod h1:WEWPzhZtxlJnov3MxcqSDiZaHHf00vs3aJwCdt3OwzA= github.com/akitasoftware/akita-libs v0.0.0-20211020162041-fe02207174fb/go.mod h1:YLFCjhwQ0ZFfYWSUD2c9KYKEeBn+R+Cz+A5SitXvJz8= -github.com/akitasoftware/akita-libs v0.0.0-20230708003852-6c8da9931921 h1:Dyka6J+ts8JEcvMwmRMll8h8Hy435Q4uaBX0odiMxe4= -github.com/akitasoftware/akita-libs v0.0.0-20230708003852-6c8da9931921/go.mod h1:qufiDcBb7r0oemPbxlXk9HUSyDt5rLO0PQGFOWRx3y4= +github.com/akitasoftware/akita-libs v0.0.0-20231110062924-14f503bf0c85 h1:QyfJXGT5/HE4IlHZrbAZlp026LeCC10gvF3Wi9BO2PA= +github.com/akitasoftware/akita-libs v0.0.0-20231110062924-14f503bf0c85/go.mod h1:EcZ7+hq27nNBm++I1C6zZ7ewnbmGoSkfdtkKw43PMKU= github.com/akitasoftware/go-utils v0.0.0-20221207014235-6f4c9079488d h1:pN1dbNacZ/mvlU1NcJVDxqmKnrDQDTVaN6iKOarfdYM= github.com/akitasoftware/go-utils v0.0.0-20221207014235-6f4c9079488d/go.mod h1:+IOXf7l/QCAQECJzjJwhTp1sBkRoJ6WciZwJezUwBa4= github.com/akitasoftware/gopacket v1.1.18-0.20210730205736-879e93dac35b h1:toBhS5rhCjo/N4YZ1cYtlsdSTGjMFH+gbJGCc+OmZiY= @@ -95,6 +95,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 h1:SjZ2GvvOononHOpK84APFuMvxqsk3tEIaKH/z4Rpu3g= github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8/go.mod h1:uEyr4WpAH4hio6LFriaPkL938XnrvLpNPmQHBdrmbIE= @@ -154,8 +155,9 @@ github.com/golang/lint v0.0.0-20170918230701-e5d664eb928e/go.mod h1:tluoj9z5200j github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -373,6 +375,9 @@ github.com/segmentio/analytics-go/v3 v3.2.1 h1:G+f90zxtc1p9G+WigVyTR0xNfOghOGs/P github.com/segmentio/analytics-go/v3 v3.2.1/go.mod h1:p8owAF8X+5o27jmvUognuXxdtqvSGtD0ZrfY2kcS9bE= github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe6qA= github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As4kEtIIOp1M= +github.com/segmentio/conf v1.2.0/go.mod h1:Y3B9O/PqqWqjyxyWWseyj/quPEtMu1zDp/kVbSWWaB0= +github.com/segmentio/go-snakecase v1.1.0/go.mod h1:jk1miR5MS7Na32PZUykG89Arm+1BUSYhuGR6b7+hJto= +github.com/segmentio/objconv v1.0.1/go.mod h1:auayaH5k3137Cl4SoXTgrzQcuQDmvuVtZgS0fb1Ahys= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -433,6 +438,8 @@ github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.0/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -447,6 +454,7 @@ golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -471,6 +479,9 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -490,7 +501,10 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -505,6 +519,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -535,7 +550,11 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -549,6 +568,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -576,6 +596,8 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -623,6 +645,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/mold.v2 v2.2.0/go.mod h1:XMyyRsGtakkDPbxXbrA5VODo6bUXyvoDjLd5l3T0XoA= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= @@ -630,6 +653,7 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/learn/parse_http.go b/learn/parse_http.go index 7b37c0ef..4cdcea74 100644 --- a/learn/parse_http.go +++ b/learn/parse_http.go @@ -130,6 +130,7 @@ func ParseHTTP(elem akinet.ParsedNetworkContent) (*PartialWitness, error) { bodyStream := rawBody.CreateReader() decodeStream, err := decodeBody(headers, bodyStream, bodyDecompressed) if err != nil { + addAgentParsingErrorToMethodMeta(methodMeta, fmt.Sprintf("Failed to decode body: %v", err)) return nil, errors.Wrap(err, "failed to decode body") } @@ -153,6 +154,7 @@ func ParseHTTP(elem akinet.ParsedNetworkContent) (*PartialWitness, error) { // https://app.clubhouse.io/akita-software/story/1898/juan-s-payload-problem telemetry.RateLimitError("unparsable body", err) printer.Debugf("skipping unparsable body: %v\n", err) + addAgentParsingErrorToMethodMeta(methodMeta, fmt.Sprintf("Failed to parse body: %v", err)) } else if bodyData != nil { datas = append(datas, bodyData) } @@ -815,3 +817,16 @@ func newDataMetaCookie(cookie *pb.HTTPCookie, responseCode int) *pb.DataMeta { } return newDataMetaHTTPMeta(m) } + +func addAgentParsingErrorToMethodMeta(meta *pb.MethodMeta, errMsg string) { + error := pb.HTTPMethodError{ + Type: pb.HTTPMethodError_AGENT_PARSING_ERROR, + Message: errMsg, + } + + if meta.Errors == nil { + meta.Errors = []*pb.HTTPMethodError{&error} + } else { + meta.Errors = append(meta.Errors, &error) + } +} diff --git a/pcap/stream.go b/pcap/stream.go index 181bff8b..9a4a7b62 100644 --- a/pcap/stream.go +++ b/pcap/stream.go @@ -55,6 +55,9 @@ type tcpFlow struct { // Context for the FIRST packet that currentParser is processing. currentParserCtx *assemblerCtxWithSeq + // Indicates if this flow has seen any packet + firstPacketSeen bool + // Data that was left unused when determining parser, awaiting for more data. // This is a hack to flush data when the flow terminates before a parser has // been selected since reassembled does not get invoked on stream end even if @@ -71,6 +74,7 @@ func newTCPFlow(clock clockWrapper, bidiID akinet.TCPBidiID, nf, tf gopacket.Flo bidiID: bidiID, outChan: outChan, factorySelector: fs, + firstPacketSeen: false, } } @@ -94,6 +98,12 @@ func (f *tcpFlow) reassembledWithIgnore(ignoreCount int, sg reassembly.ScatterGa printer.V(6).Infof("reassembled with %d bytes, isEnd=%v\n", bytesAvailable-ignoreCount, isEnd) + // since we are ending this flow, mark the firstPacketSeen as false for the current flow for the new request/response pair + // Is there a race condition here where Accept and reassembledWithIgnore will be called together for the same flow? + if isEnd { + f.firstPacketSeen = false + } + if f.currentParser == nil { // Try to create a new parser. fact, decision, discardFront := f.factorySelector.Select(pktData, isEnd) @@ -209,6 +219,7 @@ func (f *tcpFlow) reassemblyComplete() { } f.currentParser = nil f.currentParserCtx = nil + f.firstPacketSeen = false } else if f.unusedAcceptBuf.Len() > 0 { // The flow terminated before a parser has been selected, flush any bytes // that were buffered waiting for more data to determine parse. @@ -294,6 +305,14 @@ func (c *tcpStream) Accept(tcp *layers.TCP, _ gopacket.CaptureInfo, dir reassemb } } + currFlow := c.flows[dir] + revFlow := c.flows[dir.Reverse()] + + // Mark the firstPacketSeen as true for the current flow + if !currFlow.firstPacketSeen { + currFlow.firstPacketSeen = true + } + // Output some metadata for the current packet. { srcE, dstE := c.netFlow.Endpoints() @@ -314,6 +333,32 @@ func (c *tcpStream) Accept(tcp *layers.TCP, _ gopacket.CaptureInfo, dir reassemb } } + // One of the flows initiated a connection close request + if tcp.FIN { + // Confirm with mark if we need this new variable or we can achieve the same + // using revFlow.currentParser and revFlow.currentParserCtx + if !revFlow.firstPacketSeen { + currFlowParser := currFlow.currentParser + + // The current flow is the one that initiated the connection close request + // and no packets are yet arrived on the reverse flow + // this would indicate a timeout on the current side + if currFlowParser != nil { + if currFlowParser.Name() == "HTTP/1.x Request Parser Factory" { + c.outChan <- currFlow.toPNT(ac.GetCaptureInfo().Timestamp, ac.GetCaptureInfo().Timestamp, akinet.ClientTimeoutMetadata{ + StreamID: uuid.UUID(currFlow.bidiID), + Seq: int(tcp.Seq), + }) + } else if currFlowParser.Name() == "HTTP/1.x Response Parser Factory" { + c.outChan <- currFlow.toPNT(ac.GetCaptureInfo().Timestamp, ac.GetCaptureInfo().Timestamp, akinet.ServerTimeoutMetadata{ + StreamID: uuid.UUID(currFlow.bidiID), + Seq: int(tcp.Seq), + }) + } + } + } + } + // Accept everything, even if the packet might violate the TCP state machine // and get rejected by the client or server's TCP stack. We do this because we // are interested in detecting all dataflows, not just ones from valid TCP diff --git a/trace/backend_collector.go b/trace/backend_collector.go index 3174cc5c..65f8f166 100644 --- a/trace/backend_collector.go +++ b/trace/backend_collector.go @@ -179,6 +179,10 @@ func (c *BackendCollector) Process(t akinet.ParsedNetworkTraffic) error { partial, parseHTTPErr = learn.ParseHTTP(content) case akinet.HTTPResponse: partial, parseHTTPErr = learn.ParseHTTP(content) + case akinet.ClientTimeoutMetadata: + return c.processClientTimeout(t, content) + case akinet.ServerTimeoutMetadata: + return c.processServerTimeout(t, content) case akinet.TCPConnectionMetadata: return c.processTCPConnection(t, content) case akinet.TLSHandshakeMetadata: @@ -272,6 +276,64 @@ func (c *BackendCollector) processTLSHandshake(tls akinet.TLSHandshakeMetadata) return nil } +func (c *BackendCollector) processClientTimeout(packet akinet.ParsedNetworkTraffic, content akinet.ClientTimeoutMetadata) error { + witnessID := learn.ToWitnessID(content.StreamID, content.Seq) + val, ok := c.pairCache.LoadAndDelete(witnessID) + + if !ok { + // this should not have happened,a partial witness should have existed for the timeout data passed to us + return errors.Errorf("no partial witness found for client timeout %v", witnessID) + } + + pair := val.(*witnessWithInfo) + if pair.witness.Method.GetMeta() != nil { + meta := pair.witness.Method.GetMeta() + if meta != nil { + error := pb.HTTPMethodError{ + Type: pb.HTTPMethodError_CLIENT_CLOSED, + Message: "Client closed the connection", + } + + if meta.Errors == nil { + meta.Errors = []*pb.HTTPMethodError{&error} + } else { + meta.Errors = append(meta.Errors, &error) + } + } + } + + return nil +} + +func (c *BackendCollector) processServerTimeout(packet akinet.ParsedNetworkTraffic, content akinet.ServerTimeoutMetadata) error { + witnessID := learn.ToWitnessID(content.StreamID, content.Seq) + val, ok := c.pairCache.LoadAndDelete(witnessID) + + if !ok { + // this should not have happened,a partial witness should have existed for the timeout data passed to us + return errors.Errorf("no partial witness found for server timeout %v", witnessID) + } + + pair := val.(*witnessWithInfo) + if pair.witness.Method.GetMeta() != nil { + meta := pair.witness.Method.GetMeta() + if meta != nil { + error := pb.HTTPMethodError{ + Type: pb.HTTPMethodError_SERVER_CLOSED, + Message: "Server closed the connection", + } + + if meta.Errors == nil { + meta.Errors = []*pb.HTTPMethodError{&error} + } else { + meta.Errors = append(meta.Errors, &error) + } + } + } + + return nil +} + func (c *BackendCollector) queueUpload(w *witnessWithInfo) { for _, p := range c.plugins { if err := p.Transform(w.witness.GetMethod()); err != nil { From a810d149fef7669eb5d509ff2d39295fa1cb8e6a Mon Sep 17 00:00:00 2001 From: Shrey Shah Date: Fri, 10 Nov 2023 12:31:33 +0530 Subject: [PATCH 2/7] [POA-155] Fix the mock module --- go.sum | 1 + 1 file changed, 1 insertion(+) diff --git a/go.sum b/go.sum index c294c93a..dfab4d36 100644 --- a/go.sum +++ b/go.sum @@ -155,6 +155,7 @@ github.com/golang/lint v0.0.0-20170918230701-e5d664eb928e/go.mod h1:tluoj9z5200j github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= From eba7fd80d6b9e7914f2cc6bb0fcb711795c57a78 Mon Sep 17 00:00:00 2001 From: Shrey Shah Date: Wed, 15 Nov 2023 11:56:38 +0530 Subject: [PATCH 3/7] [POA-155] Address few PR comments --- learn/parse_http.go | 6 +----- pcap/stream.go | 11 ++++++----- trace/backend_collector.go | 38 ++++++++++++-------------------------- 3 files changed, 19 insertions(+), 36 deletions(-) diff --git a/learn/parse_http.go b/learn/parse_http.go index 4cdcea74..71261063 100644 --- a/learn/parse_http.go +++ b/learn/parse_http.go @@ -824,9 +824,5 @@ func addAgentParsingErrorToMethodMeta(meta *pb.MethodMeta, errMsg string) { Message: errMsg, } - if meta.Errors == nil { - meta.Errors = []*pb.HTTPMethodError{&error} - } else { - meta.Errors = append(meta.Errors, &error) - } + meta.Errors = append(meta.Errors, &error) } diff --git a/pcap/stream.go b/pcap/stream.go index 9a4a7b62..3958d234 100644 --- a/pcap/stream.go +++ b/pcap/stream.go @@ -78,6 +78,10 @@ func newTCPFlow(clock clockWrapper, bidiID akinet.TCPBidiID, nf, tf gopacket.Flo } } +func (f *tcpFlow) FirstPacketSeen() bool { + return f.firstPacketSeen +} + func (f *tcpFlow) handleUnparseable(t time.Time, size int64) { if size > 0 { f.outChan <- f.toPNT(t, t, akinet.DroppedBytes(size)) @@ -99,7 +103,6 @@ func (f *tcpFlow) reassembledWithIgnore(ignoreCount int, sg reassembly.ScatterGa printer.V(6).Infof("reassembled with %d bytes, isEnd=%v\n", bytesAvailable-ignoreCount, isEnd) // since we are ending this flow, mark the firstPacketSeen as false for the current flow for the new request/response pair - // Is there a race condition here where Accept and reassembledWithIgnore will be called together for the same flow? if isEnd { f.firstPacketSeen = false } @@ -309,7 +312,7 @@ func (c *tcpStream) Accept(tcp *layers.TCP, _ gopacket.CaptureInfo, dir reassemb revFlow := c.flows[dir.Reverse()] // Mark the firstPacketSeen as true for the current flow - if !currFlow.firstPacketSeen { + if !currFlow.FirstPacketSeen() { currFlow.firstPacketSeen = true } @@ -335,9 +338,7 @@ func (c *tcpStream) Accept(tcp *layers.TCP, _ gopacket.CaptureInfo, dir reassemb // One of the flows initiated a connection close request if tcp.FIN { - // Confirm with mark if we need this new variable or we can achieve the same - // using revFlow.currentParser and revFlow.currentParserCtx - if !revFlow.firstPacketSeen { + if !revFlow.FirstPacketSeen() { currFlowParser := currFlow.currentParser // The current flow is the one that initiated the connection close request diff --git a/trace/backend_collector.go b/trace/backend_collector.go index 65f8f166..2e02f8cc 100644 --- a/trace/backend_collector.go +++ b/trace/backend_collector.go @@ -286,20 +286,13 @@ func (c *BackendCollector) processClientTimeout(packet akinet.ParsedNetworkTraff } pair := val.(*witnessWithInfo) - if pair.witness.Method.GetMeta() != nil { - meta := pair.witness.Method.GetMeta() - if meta != nil { - error := pb.HTTPMethodError{ - Type: pb.HTTPMethodError_CLIENT_CLOSED, - Message: "Client closed the connection", - } - - if meta.Errors == nil { - meta.Errors = []*pb.HTTPMethodError{&error} - } else { - meta.Errors = append(meta.Errors, &error) - } + if meta := pair.witness.Method.GetMeta(); meta != nil { + error := pb.HTTPMethodError{ + Type: pb.HTTPMethodError_CLIENT_CLOSED, + Message: "Client closed the connection", } + + meta.Errors = append(meta.Errors, &error) } return nil @@ -315,20 +308,13 @@ func (c *BackendCollector) processServerTimeout(packet akinet.ParsedNetworkTraff } pair := val.(*witnessWithInfo) - if pair.witness.Method.GetMeta() != nil { - meta := pair.witness.Method.GetMeta() - if meta != nil { - error := pb.HTTPMethodError{ - Type: pb.HTTPMethodError_SERVER_CLOSED, - Message: "Server closed the connection", - } - - if meta.Errors == nil { - meta.Errors = []*pb.HTTPMethodError{&error} - } else { - meta.Errors = append(meta.Errors, &error) - } + if meta := pair.witness.Method.GetMeta(); meta != nil { + error := pb.HTTPMethodError{ + Type: pb.HTTPMethodError_SERVER_CLOSED, + Message: "Server closed the connection", } + + meta.Errors = append(meta.Errors, &error) } return nil From 65409f6f571d085b652d64a9307a68631b1a5b8d Mon Sep 17 00:00:00 2001 From: Shrey Shah Date: Wed, 15 Nov 2023 12:09:29 +0530 Subject: [PATCH 4/7] [POA-155] Do not error out when we dont find a partial witness for the timeout case, and remove the message field from server and client timeouts --- trace/backend_collector.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/trace/backend_collector.go b/trace/backend_collector.go index 2e02f8cc..1acb6773 100644 --- a/trace/backend_collector.go +++ b/trace/backend_collector.go @@ -281,15 +281,15 @@ func (c *BackendCollector) processClientTimeout(packet akinet.ParsedNetworkTraff val, ok := c.pairCache.LoadAndDelete(witnessID) if !ok { - // this should not have happened,a partial witness should have existed for the timeout data passed to us - return errors.Errorf("no partial witness found for client timeout %v", witnessID) + // this is probably the case where the partial witness got flushed out before we could process the timeout event + printer.Debugf("no partial witness found for client timeout event with witness id: %v\n", witnessID) + return nil } pair := val.(*witnessWithInfo) if meta := pair.witness.Method.GetMeta(); meta != nil { error := pb.HTTPMethodError{ - Type: pb.HTTPMethodError_CLIENT_CLOSED, - Message: "Client closed the connection", + Type: pb.HTTPMethodError_CLIENT_CLOSED, } meta.Errors = append(meta.Errors, &error) @@ -303,15 +303,15 @@ func (c *BackendCollector) processServerTimeout(packet akinet.ParsedNetworkTraff val, ok := c.pairCache.LoadAndDelete(witnessID) if !ok { - // this should not have happened,a partial witness should have existed for the timeout data passed to us - return errors.Errorf("no partial witness found for server timeout %v", witnessID) + // this is probably the case where the partial witness got flushed out before we could process the timeout event + printer.Debugf("no partial witness found for server timeout event with witness id: %v\n", witnessID) + return nil } pair := val.(*witnessWithInfo) if meta := pair.witness.Method.GetMeta(); meta != nil { error := pb.HTTPMethodError{ - Type: pb.HTTPMethodError_SERVER_CLOSED, - Message: "Server closed the connection", + Type: pb.HTTPMethodError_SERVER_CLOSED, } meta.Errors = append(meta.Errors, &error) From 405def1662ed014209ffb475206ebf7ba9abd8f5 Mon Sep 17 00:00:00 2001 From: Shrey Shah Date: Wed, 15 Nov 2023 21:43:42 +0530 Subject: [PATCH 5/7] [POA-155] Address akinet related PR review comments --- go.mod | 2 +- go.sum | 4 ++++ pcap/stream.go | 49 ++++++++++++++++++++++++++++++++++--------------- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index ee4309f9..c609cb3f 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/OneOfOne/xxhash v1.2.8 github.com/Pallinder/go-randomdata v1.2.0 github.com/akitasoftware/akita-ir v0.0.0-20231103112405-e2221503d639 // todo: update this to match the commit on master once akita-ir#10 is merged - github.com/akitasoftware/akita-libs v0.0.0-20231110062924-14f503bf0c85 // todo: update this to match the commit on master once akita-libs#201 is merged + github.com/akitasoftware/akita-libs v0.0.0-20231115160054-fd5262bf403b // todo: update this to match the commit on master once akita-libs#201 is merged github.com/akitasoftware/go-utils v0.0.0-20221207014235-6f4c9079488d github.com/akitasoftware/plugin-flickr v0.2.0 github.com/andybalholm/brotli v1.0.1 diff --git a/go.sum b/go.sum index dfab4d36..3f997a41 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,10 @@ github.com/akitasoftware/akita-ir v0.0.0-20231103112405-e2221503d639/go.mod h1:W github.com/akitasoftware/akita-libs v0.0.0-20211020162041-fe02207174fb/go.mod h1:YLFCjhwQ0ZFfYWSUD2c9KYKEeBn+R+Cz+A5SitXvJz8= github.com/akitasoftware/akita-libs v0.0.0-20231110062924-14f503bf0c85 h1:QyfJXGT5/HE4IlHZrbAZlp026LeCC10gvF3Wi9BO2PA= github.com/akitasoftware/akita-libs v0.0.0-20231110062924-14f503bf0c85/go.mod h1:EcZ7+hq27nNBm++I1C6zZ7ewnbmGoSkfdtkKw43PMKU= +github.com/akitasoftware/akita-libs v0.0.0-20231115144532-6bba6001976f h1:DvtaHS/cM3atMZ6yywbzAAWolMvyfgO21nreY2nm1cw= +github.com/akitasoftware/akita-libs v0.0.0-20231115144532-6bba6001976f/go.mod h1:EcZ7+hq27nNBm++I1C6zZ7ewnbmGoSkfdtkKw43PMKU= +github.com/akitasoftware/akita-libs v0.0.0-20231115160054-fd5262bf403b h1:Ed0NBTyVyrbqFa++zjlu/GnqRTWI5WBOEaWLRQAcWmQ= +github.com/akitasoftware/akita-libs v0.0.0-20231115160054-fd5262bf403b/go.mod h1:EcZ7+hq27nNBm++I1C6zZ7ewnbmGoSkfdtkKw43PMKU= github.com/akitasoftware/go-utils v0.0.0-20221207014235-6f4c9079488d h1:pN1dbNacZ/mvlU1NcJVDxqmKnrDQDTVaN6iKOarfdYM= github.com/akitasoftware/go-utils v0.0.0-20221207014235-6f4c9079488d/go.mod h1:+IOXf7l/QCAQECJzjJwhTp1sBkRoJ6WciZwJezUwBa4= github.com/akitasoftware/gopacket v1.1.18-0.20210730205736-879e93dac35b h1:toBhS5rhCjo/N4YZ1cYtlsdSTGjMFH+gbJGCc+OmZiY= diff --git a/pcap/stream.go b/pcap/stream.go index 3958d234..77c2e1db 100644 --- a/pcap/stream.go +++ b/pcap/stream.go @@ -36,6 +36,14 @@ var CountBadAssemblerContextType uint64 // bidirectional ID that identifies the tcpFlow in the opposite direction. // Writes come from TCP assembler via tcpStream, while reads come from users // of this struct. + +type tcpFlowInitiator string + +const ( + TCP_FLOW_INITIATOR_CLIENT tcpFlowInitiator = "CLIENT" + TCP_FLOW_INITIATOR_SERVER tcpFlowInitiator = "SERVER" +) + type tcpFlow struct { clock clockWrapper // constant @@ -58,6 +66,9 @@ type tcpFlow struct { // Indicates if this flow has seen any packet firstPacketSeen bool + // Indicates who initiated this flow (client or server) + initiator *tcpFlowInitiator + // Data that was left unused when determining parser, awaiting for more data. // This is a hack to flush data when the flow terminates before a parser has // been selected since reassembled does not get invoked on stream end even if @@ -82,6 +93,10 @@ func (f *tcpFlow) FirstPacketSeen() bool { return f.firstPacketSeen } +func (f *tcpFlow) TcpFlowInitiator() tcpFlowInitiator { + return *f.initiator +} + func (f *tcpFlow) handleUnparseable(t time.Time, size int64) { if size > 0 { f.outChan <- f.toPNT(t, t, akinet.DroppedBytes(size)) @@ -150,6 +165,14 @@ func (f *tcpFlow) reassembledWithIgnore(ignoreCount int, sg reassembly.ScatterGa } f.currentParser = fact.CreateParser(f.bidiID, ctx.seq, ctx.ack) f.currentParserCtx = ctx + + connectionType := f.currentParser.ConnectionType() + if connectionType == akinet.CONNECTION_TYPE_HTTP_CLIENT { + *f.initiator = TCP_FLOW_INITIATOR_CLIENT + } else if connectionType == akinet.CONNECTION_TYPE_HTTP_SERVER { + *f.initiator = TCP_FLOW_INITIATOR_SERVER + } + default: printer.Errorf("unsupported decision type %s, treating data as raw bytes\n", decision) f.handleUnparseable(sg.CaptureInfo(ignoreCount).Timestamp, pktData.Len()) @@ -339,23 +362,19 @@ func (c *tcpStream) Accept(tcp *layers.TCP, _ gopacket.CaptureInfo, dir reassemb // One of the flows initiated a connection close request if tcp.FIN { if !revFlow.FirstPacketSeen() { - currFlowParser := currFlow.currentParser - // The current flow is the one that initiated the connection close request - // and no packets are yet arrived on the reverse flow + // and no packets have yet arrived on the reverse flow // this would indicate a timeout on the current side - if currFlowParser != nil { - if currFlowParser.Name() == "HTTP/1.x Request Parser Factory" { - c.outChan <- currFlow.toPNT(ac.GetCaptureInfo().Timestamp, ac.GetCaptureInfo().Timestamp, akinet.ClientTimeoutMetadata{ - StreamID: uuid.UUID(currFlow.bidiID), - Seq: int(tcp.Seq), - }) - } else if currFlowParser.Name() == "HTTP/1.x Response Parser Factory" { - c.outChan <- currFlow.toPNT(ac.GetCaptureInfo().Timestamp, ac.GetCaptureInfo().Timestamp, akinet.ServerTimeoutMetadata{ - StreamID: uuid.UUID(currFlow.bidiID), - Seq: int(tcp.Seq), - }) - } + if currFlow.TcpFlowInitiator() == TCP_FLOW_INITIATOR_CLIENT { + c.outChan <- currFlow.toPNT(ac.GetCaptureInfo().Timestamp, ac.GetCaptureInfo().Timestamp, akinet.ClientTimeoutMetadata{ + StreamID: uuid.UUID(currFlow.bidiID), + Seq: int(tcp.Seq), + }) + } else if currFlow.TcpFlowInitiator() == TCP_FLOW_INITIATOR_SERVER { + c.outChan <- currFlow.toPNT(ac.GetCaptureInfo().Timestamp, ac.GetCaptureInfo().Timestamp, akinet.ServerTimeoutMetadata{ + StreamID: uuid.UUID(currFlow.bidiID), + Seq: int(tcp.Seq), + }) } } } From 03b07fa62c2ba705c4714677bb09fe4d431bbe2c Mon Sep 17 00:00:00 2001 From: Shrey Shah Date: Thu, 16 Nov 2023 01:15:00 +0530 Subject: [PATCH 6/7] [POA-155] Fix unit tests --- pcap/util_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pcap/util_test.go b/pcap/util_test.go index 1e85b65d..aed7243f 100644 --- a/pcap/util_test.go +++ b/pcap/util_test.go @@ -151,6 +151,10 @@ func (*princeParser) Name() string { return "prince!" } +func (*princeParser) ConnectionType() string { + return "prince!" +} + // Assumes input starts with the right header. func (p *princeParser) Parse(input memview.MemView, isEnd bool) (akinet.ParsedNetworkContent, memview.MemView, int64, error) { p.all.Append(input) @@ -217,6 +221,10 @@ func (pineappleParser) Name() string { return "pineapple!" } +func (pineappleParser) ConnectionType() string { + return "pineapple!" +} + // Assumes input starts with the right header. func (pineappleParser) Parse(input memview.MemView, isEnd bool) (akinet.ParsedNetworkContent, memview.MemView, int64, error) { return nil, memview.MemView{}, input.Len(), fmt.Errorf("should not get invoked") From d84c32c491252bc44d98c22a309cba0ae8ee8157 Mon Sep 17 00:00:00 2001 From: Shrey Shah Date: Thu, 16 Nov 2023 01:24:50 +0530 Subject: [PATCH 7/7] [POA-155] Rename the client and server timeout metadata pnc structs to client and server shutdown metadata --- go.mod | 2 +- go.sum | 2 ++ pcap/stream.go | 4 ++-- trace/backend_collector.go | 20 ++++++++++---------- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index c609cb3f..5e7f51b0 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/OneOfOne/xxhash v1.2.8 github.com/Pallinder/go-randomdata v1.2.0 github.com/akitasoftware/akita-ir v0.0.0-20231103112405-e2221503d639 // todo: update this to match the commit on master once akita-ir#10 is merged - github.com/akitasoftware/akita-libs v0.0.0-20231115160054-fd5262bf403b // todo: update this to match the commit on master once akita-libs#201 is merged + github.com/akitasoftware/akita-libs v0.0.0-20231115194858-11dd95dd76e7 // todo: update this to match the commit on master once akita-libs#201 is merged github.com/akitasoftware/go-utils v0.0.0-20221207014235-6f4c9079488d github.com/akitasoftware/plugin-flickr v0.2.0 github.com/andybalholm/brotli v1.0.1 diff --git a/go.sum b/go.sum index 3f997a41..aa9327de 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/akitasoftware/akita-libs v0.0.0-20231115144532-6bba6001976f h1:DvtaHS github.com/akitasoftware/akita-libs v0.0.0-20231115144532-6bba6001976f/go.mod h1:EcZ7+hq27nNBm++I1C6zZ7ewnbmGoSkfdtkKw43PMKU= github.com/akitasoftware/akita-libs v0.0.0-20231115160054-fd5262bf403b h1:Ed0NBTyVyrbqFa++zjlu/GnqRTWI5WBOEaWLRQAcWmQ= github.com/akitasoftware/akita-libs v0.0.0-20231115160054-fd5262bf403b/go.mod h1:EcZ7+hq27nNBm++I1C6zZ7ewnbmGoSkfdtkKw43PMKU= +github.com/akitasoftware/akita-libs v0.0.0-20231115194858-11dd95dd76e7 h1:1BeFUVT3xjJ84Ofb51r9LzWPy6cGC8VAm8txCDHxz3A= +github.com/akitasoftware/akita-libs v0.0.0-20231115194858-11dd95dd76e7/go.mod h1:EcZ7+hq27nNBm++I1C6zZ7ewnbmGoSkfdtkKw43PMKU= github.com/akitasoftware/go-utils v0.0.0-20221207014235-6f4c9079488d h1:pN1dbNacZ/mvlU1NcJVDxqmKnrDQDTVaN6iKOarfdYM= github.com/akitasoftware/go-utils v0.0.0-20221207014235-6f4c9079488d/go.mod h1:+IOXf7l/QCAQECJzjJwhTp1sBkRoJ6WciZwJezUwBa4= github.com/akitasoftware/gopacket v1.1.18-0.20210730205736-879e93dac35b h1:toBhS5rhCjo/N4YZ1cYtlsdSTGjMFH+gbJGCc+OmZiY= diff --git a/pcap/stream.go b/pcap/stream.go index 77c2e1db..a7d3732f 100644 --- a/pcap/stream.go +++ b/pcap/stream.go @@ -366,12 +366,12 @@ func (c *tcpStream) Accept(tcp *layers.TCP, _ gopacket.CaptureInfo, dir reassemb // and no packets have yet arrived on the reverse flow // this would indicate a timeout on the current side if currFlow.TcpFlowInitiator() == TCP_FLOW_INITIATOR_CLIENT { - c.outChan <- currFlow.toPNT(ac.GetCaptureInfo().Timestamp, ac.GetCaptureInfo().Timestamp, akinet.ClientTimeoutMetadata{ + c.outChan <- currFlow.toPNT(ac.GetCaptureInfo().Timestamp, ac.GetCaptureInfo().Timestamp, akinet.ClientShutdowntMetadata{ StreamID: uuid.UUID(currFlow.bidiID), Seq: int(tcp.Seq), }) } else if currFlow.TcpFlowInitiator() == TCP_FLOW_INITIATOR_SERVER { - c.outChan <- currFlow.toPNT(ac.GetCaptureInfo().Timestamp, ac.GetCaptureInfo().Timestamp, akinet.ServerTimeoutMetadata{ + c.outChan <- currFlow.toPNT(ac.GetCaptureInfo().Timestamp, ac.GetCaptureInfo().Timestamp, akinet.ServerShutdownMetadata{ StreamID: uuid.UUID(currFlow.bidiID), Seq: int(tcp.Seq), }) diff --git a/trace/backend_collector.go b/trace/backend_collector.go index 1acb6773..9758d44c 100644 --- a/trace/backend_collector.go +++ b/trace/backend_collector.go @@ -179,10 +179,10 @@ func (c *BackendCollector) Process(t akinet.ParsedNetworkTraffic) error { partial, parseHTTPErr = learn.ParseHTTP(content) case akinet.HTTPResponse: partial, parseHTTPErr = learn.ParseHTTP(content) - case akinet.ClientTimeoutMetadata: - return c.processClientTimeout(t, content) - case akinet.ServerTimeoutMetadata: - return c.processServerTimeout(t, content) + case akinet.ClientShutdowntMetadata: + return c.processClientShutdown(t, content) + case akinet.ServerShutdownMetadata: + return c.processServerShutdown(t, content) case akinet.TCPConnectionMetadata: return c.processTCPConnection(t, content) case akinet.TLSHandshakeMetadata: @@ -276,13 +276,13 @@ func (c *BackendCollector) processTLSHandshake(tls akinet.TLSHandshakeMetadata) return nil } -func (c *BackendCollector) processClientTimeout(packet akinet.ParsedNetworkTraffic, content akinet.ClientTimeoutMetadata) error { +func (c *BackendCollector) processClientShutdown(packet akinet.ParsedNetworkTraffic, content akinet.ClientShutdowntMetadata) error { witnessID := learn.ToWitnessID(content.StreamID, content.Seq) val, ok := c.pairCache.LoadAndDelete(witnessID) if !ok { - // this is probably the case where the partial witness got flushed out before we could process the timeout event - printer.Debugf("no partial witness found for client timeout event with witness id: %v\n", witnessID) + // this is probably the case where the partial witness got flushed out before we could process the shutdown event + printer.Debugf("no partial witness found for client shutdown event with witness id: %v\n", witnessID) return nil } @@ -298,13 +298,13 @@ func (c *BackendCollector) processClientTimeout(packet akinet.ParsedNetworkTraff return nil } -func (c *BackendCollector) processServerTimeout(packet akinet.ParsedNetworkTraffic, content akinet.ServerTimeoutMetadata) error { +func (c *BackendCollector) processServerShutdown(packet akinet.ParsedNetworkTraffic, content akinet.ServerShutdownMetadata) error { witnessID := learn.ToWitnessID(content.StreamID, content.Seq) val, ok := c.pairCache.LoadAndDelete(witnessID) if !ok { - // this is probably the case where the partial witness got flushed out before we could process the timeout event - printer.Debugf("no partial witness found for server timeout event with witness id: %v\n", witnessID) + // this is probably the case where the partial witness got flushed out before we could process the shutdown event + printer.Debugf("no partial witness found for server shutdown event with witness id: %v\n", witnessID) return nil }