diff --git a/c-deps/geos b/c-deps/geos index 1e320a7053da..6c5e20a1e597 160000 --- a/c-deps/geos +++ b/c-deps/geos @@ -1 +1 @@ -Subproject commit 1e320a7053da1457e723c2460c97911b26c690be +Subproject commit 6c5e20a1e59714a719b1bba2463feb5888ca42fa diff --git a/docs/RFCS/20200507_partial_indexes.md b/docs/RFCS/20200507_partial_indexes.md index f3fe4c8fe2c1..a1d0524d90d1 100644 --- a/docs/RFCS/20200507_partial_indexes.md +++ b/docs/RFCS/20200507_partial_indexes.md @@ -1,5 +1,5 @@ - Feature Name: Partial Indexes -- Status: in-progress +- Status: completed - Start Date: 2020-05-07 - Authors: mgartner - RFC PR: [#48557](https://github.com/cockroachdb/cockroach/pull/48557) @@ -400,29 +400,29 @@ disjunction on only one column, which a constraint _can_ represent. Below is a list of the steps (PRs) to implement partial indexes, roughly ordered. -- [ ] Add partial index predicate to internal index data structures, add parser +- [X] Add partial index predicate to internal index data structures, add parser support for `WHERE `, add a cluster flag for gating this defaulted to "off" -- [ ] Add simple equality implication check to optimizer when generating index +- [X] Add simple equality implication check to optimizer when generating index scans, in GenerateIndexScans. -- [ ] Same, for GenerateConstrainedScans. -- [ ] Add support for updating partial indexes on inserts. -- [ ] Add support for updating partial indexes on deletes. -- [ ] Add support for updating partial indexes on updates and upserts. -- [ ] Add support for backfilling partial indexes. -- [ ] Update the statistics builder to account for the selectivity of the partial index +- [X] Same, for GenerateConstrainedScans. +- [X] Add support for updating partial indexes on inserts. +- [X] Add support for updating partial indexes on deletes. +- [X] Add support for updating partial indexes on updates and upserts. +- [X] Add support for backfilling partial indexes. +- [X] Update the statistics builder to account for the selectivity of the partial index predicate. -- [ ] Add more advanced implication logic for filter and predicate expressions. +- [X] Add more advanced implication logic for filter and predicate expressions. - [ ] Add support in other index exploration rules: - [ ] GenerateInvertedIndexScans - [ ] GenerateZigZagJoin - [ ] GenerateInvertedIndexZigZagJoin -- [ ] Add support for partitioned partial indexes +- [X] Add support for partitioned partial indexes - [ ] Add support for using partial indexes in Lookup Joins - [ ] Consider using partial indexes for auto-generated indexes used for foreign keys. -- [ ] [Stretch goal] Add support for `ON CONFLICT WHERE [index_predicate] DO - ...` for identifying conflict behavior for uniquer partial indexes. +- [X] Add support for `ON CONFLICT WHERE [index_predicate] DO ...` for identifying + conflict behavior for unique partial indexes. - More info in the [Postgres docs](https://www.postgresql.org/docs/9.5/sql-insert.html#SQL-ON-CONFLICT) and [this blog diff --git a/go.mod b/go.mod index 9e7635c9a95f..670bdd0e551b 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,6 @@ require ( github.com/docker/distribution v2.7.0+incompatible github.com/docker/docker v17.12.0-ce-rc1.0.20190115172544-0dc531243dd3+incompatible github.com/docker/go-connections v0.4.0 - github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v1.0.0 github.com/eapache/go-resiliency v1.2.0 // indirect github.com/edsrzf/mmap-go v1.0.0 @@ -141,7 +140,7 @@ require ( github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.6.1 - github.com/twpayne/go-geom v1.3.2 + github.com/twpayne/go-geom v1.3.5 github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad github.com/zabawaba99/go-gitignore v0.0.0-20200117185801-39e6bddfb292 go.etcd.io/etcd v0.0.0-00010101000000-000000000000 @@ -163,6 +162,7 @@ require ( gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect gopkg.in/jcmturner/gokrb5.v7 v7.5.0 // indirect gopkg.in/yaml.v2 v2.3.0 + gotest.tools v2.2.0+incompatible // indirect honnef.co/go/tools v0.0.0-20190530104931-1f0868a609b7 vitess.io/vitess v0.0.0-00010101000000-000000000000 ) diff --git a/go.sum b/go.sum index 80dce8c4933b..a4f816d45cb5 100644 --- a/go.sum +++ b/go.sum @@ -121,6 +121,7 @@ github.com/broady/gogeohash v0.0.0-20120525094510-7b2c40d64042 h1:iEdmkrNMLXbM7e github.com/broady/gogeohash v0.0.0-20120525094510-7b2c40d64042/go.mod h1:f1L9YvXvlt9JTa+A17trQjSMM6bV40f+tHjB+Pi+Fqk= github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA= github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= @@ -186,7 +187,7 @@ github.com/cockroachdb/yaml v0.0.0-20180705215940-0e2822948641/go.mod h1:hI93XBm github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= -github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -222,7 +223,6 @@ github.com/docker/docker v17.12.0-ce-rc1.0.20190115172544-0dc531243dd3+incompati github.com/docker/docker v17.12.0-ce-rc1.0.20190115172544-0dc531243dd3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -366,7 +366,6 @@ github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/goware/modvendor v0.3.0 h1:pQoHt7SOUiWSwN/W5FzibTQLx/1Xa3VMBRcZGtdb1wo= github.com/goware/modvendor v0.3.0/go.mod h1:rtogeSlPLJT6MlypJyGp24o/vnHvF+ebCoTQrDX6oGY= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -507,10 +506,10 @@ github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvf github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.5-0.20190402064358-634a59d12406 h1:+OUpk+IVvmKU0jivOVFGtOzA6U5AWFs8HE4DRzWLOUE= github.com/leanovate/gopter v0.2.5-0.20190402064358-634a59d12406/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= +github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq/auth/kerberos v0.0.0-20200720160335-984a6aa1ca46 h1:q7hY+WNJTcSqJNGwJzXZYL++nWBaoKlKdgZOyY6jxz4= @@ -595,7 +594,7 @@ github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2i github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opennota/wd v0.0.0-20180911144301-b446539ab1e7 h1:cVQhwfBgiKTMAdYPbVeuIiTkdY59qZ3sp5RpyO8CNtg= github.com/opennota/wd v0.0.0-20180911144301-b446539ab1e7/go.mod h1:CS6cd3lWylVJV6EWs4Q/lkDEVGQOrEbBdwCowzzkN6A= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 h1:lM6RxxfUMrYL/f8bWEUqdXrANWtrL7Nndbm9iFN0DlU= @@ -605,7 +604,7 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/openzipkin-contrib/zipkin-go-opentracing v0.3.5 h1:82Tnq9OJpn+h5xgGpss5/mOv3KXdjtkdorFSOUusjM8= github.com/openzipkin-contrib/zipkin-go-opentracing v0.3.5/go.mod h1:uVHyebswE1cCXr2A73cRM2frx5ld1RJUCJkFNZ90ZiI= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/ory/dockertest v3.3.4+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= +github.com/ory/dockertest/v3 v3.6.0/go.mod h1:4ZOpj8qBUmh8fcBSVzkH2bws2s91JdGvHUqan4GHEuQ= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -715,11 +714,12 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/the42/cartconvert v0.0.0-20131203171324-aae784c392b8 h1:I4DY8wLxJXCrMYzDM6lKCGc3IQwJX0PlTLsd3nQqI3c= github.com/the42/cartconvert v0.0.0-20131203171324-aae784c392b8/go.mod h1:fWO/msnJVhHqN1yX6OBoxSyfj7TEj1hHiL8bJSQsK30= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/twpayne/go-geom v1.3.2 h1:4E9aSr+B5y+wjTFf2fNmxsnYSZD2k1nSdDyos3ZQ/EY= -github.com/twpayne/go-geom v1.3.2/go.mod h1:vFTJTOBkeN903mbhV6FYSTOUjglpV6WA8J11fNiUXaQ= -github.com/twpayne/go-kml v1.5.0 h1:CNWBxCmyWg1158dWbfdZsuUkUtHITrkntJS3iJhazJQ= -github.com/twpayne/go-kml v1.5.0/go.mod h1:g/OG8Q8JUxqFw8LGXE44W7osn1uXDAYaVFr1Yld43yc= +github.com/twpayne/go-geom v1.3.5 h1:VwMbubVfcMjJwpPP6a+CSte4XS1DEexfHlcrmN2D2iU= +github.com/twpayne/go-geom v1.3.5/go.mod h1:XTyWHR6+l9TUYONbbK4ImUTYbWDCu2ySSPrZmmiA0Pg= +github.com/twpayne/go-kml v1.5.1 h1:RI0JKh/VzdK/d+ZxdJzt8Ar921KMYPfg9qkw7vsbAGw= +github.com/twpayne/go-kml v1.5.1/go.mod h1:kz8jAiIz6FIdU2Zjce9qGlVtgFYES9vt7BTPBHf5jl4= github.com/twpayne/go-polyline v1.0.0/go.mod h1:ICh24bcLYBX8CknfvNPKqoTbe+eg+MX1NPyJmSBo7pU= +github.com/twpayne/go-waypoint v0.0.0-20200706203930-b263a7f6e4e8/go.mod h1:qj5pHncxKhu9gxtZEYWypA/z097sxhFlbTyOyt9gcnU= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -809,6 +809,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= @@ -853,6 +854,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200121082415-34d275377bf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -877,6 +879,7 @@ golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -959,6 +962,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/geo/encode_test.go b/pkg/geo/encode_test.go index 5e92ae7b10d3..4a96152e797a 100644 --- a/pkg/geo/encode_test.go +++ b/pkg/geo/encode_test.go @@ -47,6 +47,7 @@ func TestSpatialObjectToEWKT(t *testing.T) { }{ {"POINT(1.01 1.01)", 15, "POINT (1.01 1.01)"}, {"POINT(1.01 1.01)", 1, "POINT (1 1)"}, + {"GEOMETRYCOLLECTION (POINT EMPTY, POLYGON EMPTY)", -1, "GEOMETRYCOLLECTION (POINT EMPTY, POLYGON EMPTY)"}, {"SRID=4004;POINT(1.0 1.0)", 15, "SRID=4004;POINT (1 1)"}, } diff --git a/pkg/geo/geos/geos.cc b/pkg/geo/geos/geos.cc index f9c327a1c3d7..e823c0575ad3 100644 --- a/pkg/geo/geos/geos.cc +++ b/pkg/geo/geos/geos.cc @@ -379,49 +379,9 @@ CR_GEOS_Geometry CR_GEOS_GeometryFromSlice(CR_GEOS* lib, CR_GEOS_Handle handle, return geom; } -void CR_GEOS_PushLittleEndianUint32(std::vector& buf, uint32_t value) { - for (auto i = 0; i < sizeof(value); i++) { - buf.push_back(static_cast((value >> (8 * i)) & 0xff)); - } -} - void CR_GEOS_writeGeomToEWKB(CR_GEOS* lib, CR_GEOS_Handle handle, CR_GEOS_Geometry geom, CR_GEOS_String* ewkb, int srid) { auto hasZ = lib->GEOSHasZ_r(handle, geom); - auto isEmpty = lib->GEOSisEmpty_r(handle, geom); - // Empty points error in GEOS EWKB encoding. As such, we have to encode ourselves. - // This is still broken for GEOMETRYCOLLECTIONs/MULTIPOINT containing empty points, - // but there's not much we can do there for now. - // TODO(#geo); for 3D / 4D support, patch this to support those dimensions (we cannot extract that info with the GEOS API). - // Alternatively, upgrade GEOS to include https://github.com/libgeos/geos/commit/466cff135c8e504632ae38b79a1348dbadb390f1. - if (isEmpty && lib->GEOSGeomTypeId_r(handle, geom) == 0) { - std::vector buf; - buf.push_back('\x01'); // little endian - - uint32_t metadataInfo = 1; // 1 means it is a point. - if (srid != 0) { - metadataInfo |= 0x20000000; // add SRID bit. - } - CR_GEOS_PushLittleEndianUint32(buf, metadataInfo); - if (srid != 0) { - CR_GEOS_PushLittleEndianUint32(buf, uint32_t(srid)); - } - // Push back two NaN float values in little endian order. - for (auto i = 0; i < 2; i++) { - buf.push_back('\x00'); - buf.push_back('\x00'); - buf.push_back('\x00'); - buf.push_back('\x00'); - buf.push_back('\x00'); - buf.push_back('\x00'); - buf.push_back('\xF8'); - buf.push_back('\x7F'); - } - ewkb->data = static_cast(malloc(buf.size())); - ewkb->len = buf.size(); - memcpy(ewkb->data, buf.data(), buf.size()); - return; - } auto wkbWriter = lib->GEOSWKBWriter_create_r(handle); lib->GEOSWKBWriter_setByteOrder_r(handle, wkbWriter, 1); if (hasZ) { diff --git a/pkg/kv/kvserver/store.go b/pkg/kv/kvserver/store.go index b002b0aaaa6e..55d4b3b4ba5e 100644 --- a/pkg/kv/kvserver/store.go +++ b/pkg/kv/kvserver/store.go @@ -133,6 +133,21 @@ var concurrentRangefeedItersLimit = settings.RegisterPositiveIntSetting( 64, ) +// Minimum time interval between system config updates which will lead to +// enqueuing replicas. +var queueAdditionOnSystemConfigUpdateRate = settings.RegisterNonNegativeFloatSetting( + "kv.store.system_config_update.queue_add_rate", + "the rate (per second) at which the store will add all replicas to the split and merge queue due to system config gossip", + .5) + +// Minimum time interval between system config updates which will lead to +// enqueuing replicas. The default is relatively high to deal with startup +// scenarios. +var queueAdditionOnSystemConfigUpdateBurst = settings.RegisterNonNegativeIntSetting( + "kv.store.system_config_update.queue_add_burst", + "the burst rate at which the store will add all replicas to the split and merge queue due to system config gossip", + 32) + // raftLeadershipTransferTimeout limits the amount of time a drain command // waits for lease transfers. var raftLeadershipTransferWait = func() *settings.DurationSetting { @@ -610,7 +625,8 @@ type Store struct { // tenantRateLimiters manages tenantrate.Limiters tenantRateLimiters *tenantrate.LimiterFactory - computeInitialMetrics sync.Once + computeInitialMetrics sync.Once + systemConfigUpdateQueueRateLimiter *quotapool.RateLimiter } var _ kv.Sender = &Store{} @@ -905,6 +921,20 @@ func NewStore( s.tenantRateLimiters = tenantrate.NewLimiterFactory(cfg.Settings, &cfg.TestingKnobs.TenantRateKnobs) s.metrics.registry.AddMetricStruct(s.tenantRateLimiters.Metrics()) + s.systemConfigUpdateQueueRateLimiter = quotapool.NewRateLimiter( + "SystemConfigUpdateQueue", + quotapool.Limit(queueAdditionOnSystemConfigUpdateRate.Get(&cfg.Settings.SV)), + queueAdditionOnSystemConfigUpdateBurst.Get(&cfg.Settings.SV)) + updateSystemConfigUpdateQueueLimits := func() { + s.systemConfigUpdateQueueRateLimiter.UpdateLimit( + quotapool.Limit(queueAdditionOnSystemConfigUpdateRate.Get(&cfg.Settings.SV)), + queueAdditionOnSystemConfigUpdateBurst.Get(&cfg.Settings.SV)) + } + queueAdditionOnSystemConfigUpdateRate.SetOnChange(&cfg.Settings.SV, + updateSystemConfigUpdateQueueLimits) + queueAdditionOnSystemConfigUpdateBurst.SetOnChange(&cfg.Settings.SV, + updateSystemConfigUpdateQueueLimits) + if s.cfg.Gossip != nil { // Add range scanner and configure with queues. s.scanner = newReplicaScanner( @@ -1826,6 +1856,7 @@ func (s *Store) systemGossipUpdate(sysCfg *config.SystemConfig) { // For every range, update its zone config and check if it needs to // be split or merged. now := s.cfg.Clock.Now() + shouldQueue := s.systemConfigUpdateQueueRateLimiter.AdmitN(1) newStoreReplicaVisitor(s).Visit(func(repl *Replica) bool { key := repl.Desc().StartKey zone, err := sysCfg.GetZoneConfigForKey(key) @@ -1836,12 +1867,14 @@ func (s *Store) systemGossipUpdate(sysCfg *config.SystemConfig) { zone = s.cfg.DefaultZoneConfig } repl.SetZoneConfig(zone) - s.splitQueue.Async(ctx, "gossip update", true /* wait */, func(ctx context.Context, h queueHelper) { - h.MaybeAdd(ctx, repl, now) - }) - s.mergeQueue.Async(ctx, "gossip update", true /* wait */, func(ctx context.Context, h queueHelper) { - h.MaybeAdd(ctx, repl, now) - }) + if shouldQueue { + s.splitQueue.Async(ctx, "gossip update", true /* wait */, func(ctx context.Context, h queueHelper) { + h.MaybeAdd(ctx, repl, now) + }) + s.mergeQueue.Async(ctx, "gossip update", true /* wait */, func(ctx context.Context, h queueHelper) { + h.MaybeAdd(ctx, repl, now) + }) + } return true // more }) } diff --git a/pkg/sql/logictest/testdata/logic_test/geospatial b/pkg/sql/logictest/testdata/logic_test/geospatial index 4c5c8e53fe57..73d765148660 100644 --- a/pkg/sql/logictest/testdata/logic_test/geospatial +++ b/pkg/sql/logictest/testdata/logic_test/geospatial @@ -2933,10 +2933,10 @@ ORDER BY a.dsc, b.fraction, c.repeat ---- Empty LineString 0.0 false POINT EMPTY POINT EMPTY POINT EMPTY Empty LineString 0.0 true POINT EMPTY POINT EMPTY POINT EMPTY -Empty LineString 0.2 false POINT EMPTY MULTIPOINT EMPTY POINT EMPTY -Empty LineString 0.2 true POINT EMPTY MULTIPOINT EMPTY MULTIPOINT EMPTY -Empty LineString 0.5 false POINT EMPTY MULTIPOINT EMPTY POINT EMPTY -Empty LineString 0.5 true POINT EMPTY MULTIPOINT EMPTY MULTIPOINT EMPTY +Empty LineString 0.2 false POINT EMPTY MULTIPOINT (EMPTY, EMPTY, EMPTY, EMPTY, EMPTY) POINT EMPTY +Empty LineString 0.2 true POINT EMPTY MULTIPOINT (EMPTY, EMPTY, EMPTY, EMPTY, EMPTY) MULTIPOINT (EMPTY, EMPTY, EMPTY, EMPTY, EMPTY) +Empty LineString 0.5 false POINT EMPTY MULTIPOINT (EMPTY, EMPTY) POINT EMPTY +Empty LineString 0.5 true POINT EMPTY MULTIPOINT (EMPTY, EMPTY) MULTIPOINT (EMPTY, EMPTY) Empty LineString 0.51 false POINT EMPTY POINT EMPTY POINT EMPTY Empty LineString 0.51 true POINT EMPTY POINT EMPTY POINT EMPTY Empty LineString 1.0 false POINT EMPTY POINT EMPTY POINT EMPTY diff --git a/pkg/util/quotapool/int_rate_impl_test.go b/pkg/util/quotapool/int_rate.go similarity index 69% rename from pkg/util/quotapool/int_rate_impl_test.go rename to pkg/util/quotapool/int_rate.go index 0f823f3947ab..544488110373 100644 --- a/pkg/util/quotapool/int_rate_impl_test.go +++ b/pkg/util/quotapool/int_rate.go @@ -16,6 +16,7 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/util/timeutil" + "github.com/cockroachdb/errors" ) // Limit defines a rate in terms of quota per second. @@ -26,10 +27,6 @@ type Limit float64 // in the case that they end up not getting used. type RateLimiter struct { qp *QuotaPool - - // TODO(ajwerner): synchronization around changing limits. - burst int64 - rateLimit Limit } // NewRateLimiter defines a new RateLimiter. The limiter is implemented as a @@ -37,8 +34,12 @@ type RateLimiter struct { // acquire more than burst, it will block until the bucket is full and then // put the token bucket in debt. func NewRateLimiter(name string, rate Limit, burst int64, options ...Option) *RateLimiter { - rl := &RateLimiter{rateLimit: rate, burst: burst} + rl := &RateLimiter{} bucket := rateBucket{ + limitConfig: limitConfig{ + rate: rate, + burst: burst, + }, p: rl, cur: float64(burst), lastUpdated: timeutil.Now(), @@ -72,24 +73,61 @@ func (rl *RateLimiter) WaitN(ctx context.Context, n int64) error { return nil } +// AdmitN acquire n quota from the RateLimiter if it succeeds. It will return +// false and not block if there is currently insufficient quota or the pool is +// closed. +func (rl *RateLimiter) AdmitN(n int64) bool { + r := rl.newRateRequest(n) + defer rl.putRateRequest(r) + return rl.qp.Acquire(context.Background(), (*rateRequestNoWait)(r)) == nil +} + +// UpdateLimit updates the rate and burst limits. The change in burst will +// be applied to the current quantity of quota. For example, if the RateLimiter +// currently had a quota of 5 available with a burst of 10 and the burst is +// update to 20, the quota will increase to 15. Similarly, if the burst is +// decreased by 10, the current quota will decrease accordingly, potentially +// putting the limiter into debt. +func (rl *RateLimiter) UpdateLimit(rate Limit, burst int64) { + cfg := limitConfig{rate: rate, burst: burst} + rl.qp.Add(&cfg) +} + // rateBucket is the implementation of Resource which remains in the quotapool // for a RateLimiter. type rateBucket struct { + limitConfig p *RateLimiter cur float64 lastUpdated time.Time } +type limitConfig struct { + rate Limit + burst int64 +} + var _ Resource = (*rateBucket)(nil) -func (i *rateBucket) Merge(val interface{}) (shouldNotify bool) { - v := val.(*rateAlloc) - i.cur += float64(v.alloc) - v.rl.putRateAlloc(v) - if i.cur > float64(i.p.burst) { - i.cur = float64(i.p.burst) +func (r *rateBucket) Merge(v interface{}) (shouldNotify bool) { + switch v := v.(type) { + case *rateAlloc: + r.cur += float64(v.alloc) + v.rl.putRateAlloc(v) + if r.cur > float64(r.burst) { + r.cur = float64(r.burst) + } + return true + case *limitConfig: + shouldNotify = r.burst < v.burst || r.rate < v.rate + burstDelta := v.burst - r.burst + r.limitConfig = *v + r.cur += float64(burstDelta) + r.update(r.p.qp.TimeSource().Now()) + return shouldNotify + default: + panic(errors.Errorf("unexpected merge value type %T", v)) } - return true } // RateAlloc is an allocated quantity of quota which can be released back into @@ -141,27 +179,19 @@ func (i *rateRequest) Acquire( r := res.(*rateBucket) now := r.p.qp.timeSource.Now() - // TODO(ajwerner): Consider instituting a minimum update frequency to avoid - // spinning too fast on timers for tons of tiny allocations at a fast rate. - if since := now.Sub(r.lastUpdated); since > 0 { - r.cur += float64(r.p.rateLimit) * since.Seconds() - if r.cur > float64(r.p.burst) { - r.cur = float64(r.p.burst) - } - r.lastUpdated = now - } + r.update(now) // Deal with the case where the allocation is larger than the burst size. // In this case we'll allow the acquisition to complete if the current value // is equal to the burst. If the acquisition succeeds, it will put the limiter // into debt. want := float64(i.want) - if i.want > r.p.burst { - want = float64(r.p.burst) + if i.want > r.burst { + want = float64(r.burst) } if delta := want - r.cur; delta > 0 { // Compute the time it will take for r.cur to get to the needed capacity. - timeDelta := time.Duration((delta * float64(time.Second)) / float64(r.p.rateLimit)) + timeDelta := time.Duration((delta * float64(time.Second)) / float64(r.rate)) // Deal with the exceedingly edge case that timeDelta, as a floating point // number, is less than 1ns by returning 1ns and looping back around. @@ -175,6 +205,18 @@ func (i *rateRequest) Acquire( return true, 0 } +func (r *rateBucket) update(now time.Time) { + // TODO(ajwerner): Consider instituting a minimum update frequency to avoid + // spinning too fast on timers for tons of tiny allocations at a fast rate. + if since := now.Sub(r.lastUpdated); since > 0 { + r.cur += float64(r.rate) * since.Seconds() + if r.cur > float64(r.burst) { + r.cur = float64(r.burst) + } + r.lastUpdated = now + } +} + func (i *rateRequest) ShouldWait() bool { return true } @@ -193,3 +235,19 @@ func (rl *RateLimiter) putRateAlloc(a *rateAlloc) { *a = rateAlloc{} rateAllocSyncPool.Put(a) } + +// rateRequestNoWait is like a rate request but will not block waiting for +// quota. +type rateRequestNoWait rateRequest + +func (r *rateRequestNoWait) Acquire( + ctx context.Context, resource Resource, +) (fulfilled bool, tryAgainAfter time.Duration) { + return (*rateRequest)(r).Acquire(ctx, resource) +} + +func (r *rateRequestNoWait) ShouldWait() bool { + return false +} + +var _ Request = (*rateRequestNoWait)(nil) diff --git a/pkg/util/quotapool/int_rate_test.go b/pkg/util/quotapool/int_rate_test.go index cc735460889b..5f605894eade 100644 --- a/pkg/util/quotapool/int_rate_test.go +++ b/pkg/util/quotapool/int_rate_test.go @@ -140,6 +140,58 @@ func TestRateLimiterBasic(t *testing.T) { mt.Advance(2 * time.Millisecond) <-done } + { + // Fill the bucket all the way up. + mt.Advance(2 * time.Second) + + // Consume some. There should be 10 in the bucket. + go doWait(10) + <-done + + // THis should need to wait one second for the bucket to fill up. + go doWait(30) + ensureNotDone() + + // Adjust the rate and the burst down. This should move the current + // capacity down to 0 and lower the burst. It will now take 10 seconds + // before the bucket is full. + rl.UpdateLimit(1, 10) + ensureNotDone() + mt.Advance(9 * time.Second) + ensureNotDone() + mt.Advance(time.Second) + <-done + + // At this point, the limiter should be 20 in debt so it should take + // 20s before the current goroutine is unblocked. + go doWait(2) + ensureNotDone() + + // Adjust the rate and burst up. The burst delta is 10, so the debt should + // reduce to 10 and the rate is doubled. In 6 seconds the goroutine should + // unblock. + rl.UpdateLimit(2, 20) + mt.Advance(5 * time.Second) + ensureNotDone() + mt.Advance(time.Second) + <-done + + // Set the limit and burst back to the default values. + rl.UpdateLimit(10, 20) + } + { + // Fill the bucket all the way up. + mt.Advance(2 * time.Second) + + // Consume some. There should be 10 in the bucket. + go doWait(10) + <-done + + require.False(t, rl.AdmitN(11)) + require.True(t, rl.AdmitN(9)) // 1 left + require.False(t, rl.AdmitN(2)) + require.True(t, rl.AdmitN(1)) // 0 left + } } // TestRateLimitWithVerySmallDelta ensures that in cases where the delta is diff --git a/vendor b/vendor index 0f7f5a4876dd..42efe9fa6bfa 160000 --- a/vendor +++ b/vendor @@ -1 +1 @@ -Subproject commit 0f7f5a4876dd15c29b8dfc9161c0ee06dd566f0e +Subproject commit 42efe9fa6bfa3354f9c62bdea44c25b34549ea4e