From 0c1db147aaa9903be154b82436b55a9cf2e86b49 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Thu, 17 May 2018 04:48:57 -0400 Subject: [PATCH 01/54] ID extractor enhancements: extended tests, benchmarks and BodyExtractor fix (#1695) * Enable ID extractor testing during normal builds, rename the Python test file * Implement BaseExtractor.ExtractBody, remove unused XPathExtractorConfig fields * Extend ID extractor tests, add benchmarks --- coprocess_dummy.go | 2 - coprocess_id_extractor.go | 19 +- coprocess_id_extractor_python_test.go | 203 +++++++++ coprocess_id_extractor_test.go | 595 ++++++++++++++++++-------- 4 files changed, 621 insertions(+), 198 deletions(-) create mode 100644 coprocess_id_extractor_python_test.go diff --git a/coprocess_dummy.go b/coprocess_dummy.go index d6afa43a97e..8dd14755796 100644 --- a/coprocess_dummy.go +++ b/coprocess_dummy.go @@ -61,5 +61,3 @@ func CoProcessInit() error { } func doCoprocessReload() {} - -func newExtractor(referenceSpec *APISpec, mw BaseMiddleware) {} diff --git a/coprocess_id_extractor.go b/coprocess_id_extractor.go index d620fe44ed9..20408caa9cf 100644 --- a/coprocess_id_extractor.go +++ b/coprocess_id_extractor.go @@ -1,11 +1,10 @@ -// +build coprocess - package main import ( "crypto/md5" "errors" "fmt" + "io/ioutil" "net/http" "regexp" "strings" @@ -77,8 +76,14 @@ func (e *BaseExtractor) ExtractForm(r *http.Request, paramName string) (formValu return strings.Join(values, ""), nil } +// ExtractBody is used when BodySource is specified. func (e *BaseExtractor) ExtractBody(r *http.Request) (bodyValue string, err error) { - return bodyValue, err + copiedRequest := copyRequest(r) + body, err := ioutil.ReadAll(copiedRequest.Body) + if err != nil { + return bodyValue, err + } + return string(body), err } // Error is a helper for logging the extractor errors. It always returns HTTP 400 (so we don't expose any details). @@ -200,7 +205,6 @@ func (e *RegexExtractor) ExtractAndCheck(r *http.Request) (SessionID string, ret } regexOutput := expression.FindAllString(extractorOutput, -1) - if config.RegexMatchIndex > len(regexOutput)-1 { returnOverrides = e.Error(r, fmt.Errorf("Can't find regexp match group"), "RegexExtractor error") return SessionID, returnOverrides @@ -228,10 +232,8 @@ type XPathExtractor struct { } type XPathExtractorConfig struct { - HeaderName string `mapstructure:"header_name" bson:"header_name" json:"header_name"` - RegexExpression string `mapstructure:"regex_expression" bson:"regex_expression" json:"regex_expression"` - RegexMatchIndex int `mapstructure:"regex_match_index" bson:"regex_match_index" json:"regex_match_index"` - FormParamName string `mapstructure:"param_name" bson:"param_name" json:"param_name"` + HeaderName string `mapstructure:"header_name" bson:"header_name" json:"header_name"` + FormParamName string `mapstructure:"param_name" bson:"param_name" json:"param_name"` } func (e *XPathExtractor) ExtractAndCheck(r *http.Request) (SessionID string, returnOverrides ReturnOverrides) { @@ -246,7 +248,6 @@ func (e *XPathExtractor) ExtractAndCheck(r *http.Request) (SessionID string, ret } expressionString := e.Config.ExtractorConfig["xpath_expression"].(string) - expression, err := xmlpath.Compile(expressionString) if err != nil { returnOverrides = e.Error(r, err, "XPathExtractor: bad expression") diff --git a/coprocess_id_extractor_python_test.go b/coprocess_id_extractor_python_test.go new file mode 100644 index 00000000000..9715f3201cf --- /dev/null +++ b/coprocess_id_extractor_python_test.go @@ -0,0 +1,203 @@ +// +build coprocess +// +build python + +package main + +import ( + "net/url" + "testing" + "time" + + "github.com/TykTechnologies/tyk/test" +) + +var pythonIDExtractorHeaderValue = map[string]string{ + "manifest.json": ` + { + "file_list": [ + "middleware.py" + ], + "custom_middleware": { + "driver": "python", + "auth_check": { + "name": "MyAuthHook" + }, + "id_extractor": { + "extract_from": "header", + "extract_with": "value", + "extractor_config": { + "header_name": "Authorization" + } + } + } + } + `, + "middleware.py": ` +import time +from tyk.decorators import * +from gateway import TykGateway as tyk + +counter = 0 + +@Hook +def MyAuthHook(request, session, metadata, spec): + global counter + counter = counter + 1 + auth_header = request.get_header('Authorization') + if auth_header == 'valid_token' and counter < 2: + session.rate = 1000.0 + session.per = 1.0 + session.id_extractor_deadline = int(time.time()) + 60 + metadata["token"] = "valid_token" + return request, session, metadata + `, +} + +var pythonIDExtractorFormValue = map[string]string{ + "manifest.json": ` + { + "file_list": [ + "middleware.py" + ], + "custom_middleware": { + "driver": "python", + "auth_check": { + "name": "MyAuthHook" + }, + "id_extractor": { + "extract_from": "form", + "extract_with": "value", + "extractor_config": { + "param_name": "auth" + } + } + } + } + `, + "middleware.py": ` +import time +from tyk.decorators import * +from gateway import TykGateway as tyk +from urllib import parse + +counter = 0 + +@Hook +def MyAuthHook(request, session, metadata, spec): + global counter + counter = counter + 1 + auth_param = parse.parse_qs(request.object.body)["auth"] + if auth_param and auth_param[0] == 'valid_token' and counter < 2: + session.rate = 1000.0 + session.per = 1.0 + session.id_extractor_deadline = int(time.time()) + 60 + metadata["token"] = "valid_token" + return request, session, metadata +`, +} + +var pythonIDExtractorHeaderRegex = map[string]string{ + "manifest.json": ` + { + "file_list": [ + "middleware.py" + ], + "custom_middleware": { + "driver": "python", + "auth_check": { + "name": "MyAuthHook" + }, + "id_extractor": { + "extract_from": "header", + "extract_with": "regex", + "extractor_config": { + "header_name": "Authorization", + "regex_expression": "[0-9]+", + "regex_match_index": 0 + } + } + } + } + `, + "middleware.py": ` +import time +from tyk.decorators import * +from gateway import TykGateway as tyk + +counter = 0 + +@Hook +def MyAuthHook(request, session, metadata, spec): + print("MyAuthHook3 is called") + global counter + counter = counter + 1 + _, auth_header = request.get_header('Authorization').split('-') + if auth_header and auth_header == '12345' and counter < 2: + session.rate = 1000.0 + session.per = 1.0 + session.id_extractor_deadline = int(time.time()) + 60 + metadata["token"] = "valid_token" + return request, session, metadata + `, +} + +/* Value Extractor tests, using "header" source */ +// Goal of ID extractor is to cache auth plugin calls +// Our `pythonBundleWithAuthCheck` plugin restrict more then 1 call +// With ID extractor, it should run multiple times (because cache) +func TestValueExtractorHeaderSource(t *testing.T) { + ts := newTykTestServer(tykTestServerConfig{ + delay: 10 * time.Millisecond, + }) + defer ts.Close() + + spec := buildAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.UseKeylessAccess = false + spec.EnableCoProcessAuth = true + })[0] + t.Run("Header value", func(t *testing.T) { + bundleID := registerBundle("id_extractor_header_value", pythonIDExtractorHeaderValue) + spec.CustomMiddlewareBundle = bundleID + spec.APIID = "api1" + + loadAPI(spec) + time.Sleep(1 * time.Second) + + ts.Run(t, []test.TestCase{ + {Path: "/", Headers: map[string]string{"Authorization": "valid_token"}, Code: 200}, + {Path: "/", Headers: map[string]string{"Authorization": "valid_token"}, Code: 200}, + {Path: "/", Headers: map[string]string{"Authorization": "invalid_token"}, Code: 403}, + }...) + }) + t.Run("Form value", func(t *testing.T) { + bundleID := registerBundle("id_extractor_form_value", pythonIDExtractorFormValue) + spec.CustomMiddlewareBundle = bundleID + spec.APIID = "api2" + + loadAPI(spec) + time.Sleep(1 * time.Second) + + formHeaders := map[string]string{"Content-Type": "application/x-www-form-urlencoded"} + + ts.Run(t, []test.TestCase{ + {Method: "POST", Path: "/", Headers: formHeaders, Data: url.Values{"auth": []string{"valid_token"}}.Encode(), Code: 200}, + {Method: "POST", Path: "/", Headers: formHeaders, Data: url.Values{"auth": []string{"valid_token"}}.Encode(), Code: 200}, + {Method: "POST", Path: "/", Headers: formHeaders, Data: url.Values{"auth": []string{"invalid_token"}}.Encode(), Code: 403}, + }...) + }) + t.Run("Header regex", func(t *testing.T) { + bundleID := registerBundle("id_extractor_header_regex", pythonIDExtractorHeaderRegex) + spec.CustomMiddlewareBundle = bundleID + spec.APIID = "api3" + + loadAPI(spec) + time.Sleep(1 * time.Second) + + ts.Run(t, []test.TestCase{ + {Path: "/", Headers: map[string]string{"Authorization": "prefix-12345"}, Code: 200}, + {Path: "/", Headers: map[string]string{"Authorization": "prefix-12345"}, Code: 200}, + {Path: "/", Headers: map[string]string{"Authorization": "prefix-123456"}, Code: 403}, + }...) + }) +} diff --git a/coprocess_id_extractor_test.go b/coprocess_id_extractor_test.go index 9715f3201cf..e6410acd056 100644 --- a/coprocess_id_extractor_test.go +++ b/coprocess_id_extractor_test.go @@ -1,203 +1,424 @@ -// +build coprocess -// +build python - package main import ( + "crypto/md5" + "fmt" + "net/http" "net/url" + "strings" "testing" - "time" - "github.com/TykTechnologies/tyk/test" + "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/storage" +) + +const ( + extractorTestOrgID = "testorg" + + extractorValueInput = "testkey" + + extractorRegexExpr = "prefix-(.*)" + extractorRegexInput = "prefix-testkey123" + extractorRegexMatchIndex = 0 + + extractorXPathExpr = "//object/key" + extractorXPathInput = "thevalue" + + extractorHeaderName = "testheader" + extractorParamName = "testparam" ) -var pythonIDExtractorHeaderValue = map[string]string{ - "manifest.json": ` - { - "file_list": [ - "middleware.py" - ], - "custom_middleware": { - "driver": "python", - "auth_check": { - "name": "MyAuthHook" - }, - "id_extractor": { - "extract_from": "header", - "extract_with": "value", - "extractor_config": { - "header_name": "Authorization" - } - } - } - } - `, - "middleware.py": ` -import time -from tyk.decorators import * -from gateway import TykGateway as tyk - -counter = 0 - -@Hook -def MyAuthHook(request, session, metadata, spec): - global counter - counter = counter + 1 - auth_header = request.get_header('Authorization') - if auth_header == 'valid_token' and counter < 2: - session.rate = 1000.0 - session.per = 1.0 - session.id_extractor_deadline = int(time.time()) + 60 - metadata["token"] = "valid_token" - return request, session, metadata - `, +func createSpecTestFrom(t testing.TB, def *apidef.APIDefinition) *APISpec { + loader := APIDefinitionLoader{} + spec := loader.MakeSpec(def) + tname := t.Name() + redisStore := storage.RedisCluster{KeyPrefix: tname + "-apikey."} + healthStore := storage.RedisCluster{KeyPrefix: tname + "-apihealth."} + orgStore := storage.RedisCluster{KeyPrefix: tname + "-orgKey."} + spec.Init(redisStore, redisStore, healthStore, orgStore) + return spec +} + +func prepareExtractor(t testing.TB, extractorSource apidef.IdExtractorSource, extractorType apidef.IdExtractorType, config map[string]interface{}) (IdExtractor, *APISpec) { + def := &apidef.APIDefinition{ + OrgID: extractorTestOrgID, + CustomMiddleware: apidef.MiddlewareSection{ + IdExtractor: apidef.MiddlewareIdExtractor{ + ExtractFrom: extractorSource, + ExtractWith: extractorType, + ExtractorConfig: config, + }, + }, + } + spec := createSpecTestFrom(t, def) + mw := BaseMiddleware{Spec: spec} + newExtractor(spec, mw) + return spec.CustomMiddleware.IdExtractor.Extractor.(IdExtractor), spec +} + +func prepareHeaderExtractorRequest(headers map[string]string) *http.Request { + r, _ := http.NewRequest(http.MethodGet, "/", nil) + if headers == nil { + return r + } + for k, v := range headers { + r.Header.Add(k, v) + } + return r +} + +func prepareBodyExtractorRequest(input string) *http.Request { + r, _ := http.NewRequest(http.MethodPost, "/", strings.NewReader(input)) + return r +} + +func prepareExtractorFormRequest(values map[string]string) *http.Request { + formData := url.Values{} + if values != nil { + for k, v := range values { + formData.Set(k, v) + } + } + r, _ := http.NewRequest(http.MethodPost, "/", strings.NewReader(formData.Encode())) + r.Header.Add("Content-Type", "application/x-www-form-urlencoded") + return r } -var pythonIDExtractorFormValue = map[string]string{ - "manifest.json": ` - { - "file_list": [ - "middleware.py" - ], - "custom_middleware": { - "driver": "python", - "auth_check": { - "name": "MyAuthHook" - }, - "id_extractor": { - "extract_from": "form", - "extract_with": "value", - "extractor_config": { - "param_name": "auth" - } - } - } - } - `, - "middleware.py": ` -import time -from tyk.decorators import * -from gateway import TykGateway as tyk -from urllib import parse - -counter = 0 - -@Hook -def MyAuthHook(request, session, metadata, spec): - global counter - counter = counter + 1 - auth_param = parse.parse_qs(request.object.body)["auth"] - if auth_param and auth_param[0] == 'valid_token' and counter < 2: - session.rate = 1000.0 - session.per = 1.0 - session.id_extractor_deadline = int(time.time()) + 60 - metadata["token"] = "valid_token" - return request, session, metadata -`, +func generateSessionID(input string) string { + data := []byte(input) + tokenID := fmt.Sprintf("%x", md5.Sum(data)) + return extractorTestOrgID + tokenID } -var pythonIDExtractorHeaderRegex = map[string]string{ - "manifest.json": ` - { - "file_list": [ - "middleware.py" - ], - "custom_middleware": { - "driver": "python", - "auth_check": { - "name": "MyAuthHook" - }, - "id_extractor": { - "extract_from": "header", - "extract_with": "regex", - "extractor_config": { - "header_name": "Authorization", - "regex_expression": "[0-9]+", - "regex_match_index": 0 - } - } - } - } - `, - "middleware.py": ` -import time -from tyk.decorators import * -from gateway import TykGateway as tyk - -counter = 0 - -@Hook -def MyAuthHook(request, session, metadata, spec): - print("MyAuthHook3 is called") - global counter - counter = counter + 1 - _, auth_header = request.get_header('Authorization').split('-') - if auth_header and auth_header == '12345' and counter < 2: - session.rate = 1000.0 - session.per = 1.0 - session.id_extractor_deadline = int(time.time()) + 60 - metadata["token"] = "valid_token" - return request, session, metadata - `, +func TestValueExtractor(t *testing.T) { + testSessionID := generateSessionID(extractorValueInput) + + t.Run("HeaderSource", func(t *testing.T) { + extractor, spec := prepareExtractor(t, apidef.HeaderSource, apidef.ValueExtractor, map[string]interface{}{ + "header_name": extractorHeaderName, + }) + + r := prepareHeaderExtractorRequest(nil) + sessionID, overrides := extractor.ExtractAndCheck(r) + if sessionID != "" { + t.Fatalf("should return an empty session ID, got %s", sessionID) + } + if overrides.ResponseCode != 400 { + t.Fatalf("should return 400, got %d", overrides.ResponseCode) + } + + r = prepareHeaderExtractorRequest(map[string]string{extractorHeaderName: extractorValueInput}) + sessionID, overrides = extractor.ExtractAndCheck(r) + if sessionID != testSessionID { + t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) + } + if !strings.HasPrefix(sessionID, spec.OrgID) { + t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) + } + if overrides.ResponseCode != 0 { + t.Fatalf("response code should be 0, got %d", overrides.ResponseCode) + } + }) + + t.Run("FormSource", func(t *testing.T) { + extractor, spec := prepareExtractor(t, apidef.FormSource, apidef.ValueExtractor, map[string]interface{}{ + "param_name": extractorParamName, + }) + + r := prepareExtractorFormRequest(nil) + sessionID, overrides := extractor.ExtractAndCheck(r) + if sessionID != "" { + t.Fatalf("should return an empty session ID, got %s", sessionID) + } + if overrides.ResponseCode != 400 { + t.Fatalf("should return 400, got %d", overrides.ResponseCode) + } + + r = prepareExtractorFormRequest(map[string]string{extractorParamName: extractorValueInput}) + sessionID, overrides = extractor.ExtractAndCheck(r) + if sessionID != testSessionID { + t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) + } + if !strings.HasPrefix(sessionID, spec.OrgID) { + t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) + } + if overrides.ResponseCode != 0 { + t.Fatalf("response code should be 0, got %d", overrides.ResponseCode) + } + }) } -/* Value Extractor tests, using "header" source */ -// Goal of ID extractor is to cache auth plugin calls -// Our `pythonBundleWithAuthCheck` plugin restrict more then 1 call -// With ID extractor, it should run multiple times (because cache) -func TestValueExtractorHeaderSource(t *testing.T) { - ts := newTykTestServer(tykTestServerConfig{ - delay: 10 * time.Millisecond, - }) - defer ts.Close() - - spec := buildAPI(func(spec *APISpec) { - spec.Proxy.ListenPath = "/" - spec.UseKeylessAccess = false - spec.EnableCoProcessAuth = true - })[0] - t.Run("Header value", func(t *testing.T) { - bundleID := registerBundle("id_extractor_header_value", pythonIDExtractorHeaderValue) - spec.CustomMiddlewareBundle = bundleID - spec.APIID = "api1" - - loadAPI(spec) - time.Sleep(1 * time.Second) - - ts.Run(t, []test.TestCase{ - {Path: "/", Headers: map[string]string{"Authorization": "valid_token"}, Code: 200}, - {Path: "/", Headers: map[string]string{"Authorization": "valid_token"}, Code: 200}, - {Path: "/", Headers: map[string]string{"Authorization": "invalid_token"}, Code: 403}, - }...) - }) - t.Run("Form value", func(t *testing.T) { - bundleID := registerBundle("id_extractor_form_value", pythonIDExtractorFormValue) - spec.CustomMiddlewareBundle = bundleID - spec.APIID = "api2" - - loadAPI(spec) - time.Sleep(1 * time.Second) - - formHeaders := map[string]string{"Content-Type": "application/x-www-form-urlencoded"} - - ts.Run(t, []test.TestCase{ - {Method: "POST", Path: "/", Headers: formHeaders, Data: url.Values{"auth": []string{"valid_token"}}.Encode(), Code: 200}, - {Method: "POST", Path: "/", Headers: formHeaders, Data: url.Values{"auth": []string{"valid_token"}}.Encode(), Code: 200}, - {Method: "POST", Path: "/", Headers: formHeaders, Data: url.Values{"auth": []string{"invalid_token"}}.Encode(), Code: 403}, - }...) - }) - t.Run("Header regex", func(t *testing.T) { - bundleID := registerBundle("id_extractor_header_regex", pythonIDExtractorHeaderRegex) - spec.CustomMiddlewareBundle = bundleID - spec.APIID = "api3" - - loadAPI(spec) - time.Sleep(1 * time.Second) - - ts.Run(t, []test.TestCase{ - {Path: "/", Headers: map[string]string{"Authorization": "prefix-12345"}, Code: 200}, - {Path: "/", Headers: map[string]string{"Authorization": "prefix-12345"}, Code: 200}, - {Path: "/", Headers: map[string]string{"Authorization": "prefix-123456"}, Code: 403}, - }...) +func TestRegexExtractor(t *testing.T) { + testSessionID := generateSessionID(extractorRegexInput) + + t.Run("HeaderSource", func(t *testing.T) { + extractor, spec := prepareExtractor(t, apidef.HeaderSource, apidef.RegexExtractor, map[string]interface{}{ + "regex_expression": extractorRegexExpr, + "regex_match_index": extractorRegexMatchIndex, + "header_name": extractorHeaderName, + }) + + r := prepareHeaderExtractorRequest(nil) + sessionID, overrides := extractor.ExtractAndCheck(r) + if sessionID != "" { + t.Fatalf("should return an empty session ID, got %s", sessionID) + } + if overrides.ResponseCode != 400 { + t.Fatalf("should return 400, got %d", overrides.ResponseCode) + } + + r = prepareHeaderExtractorRequest(map[string]string{extractorHeaderName: extractorRegexInput}) + sessionID, overrides = extractor.ExtractAndCheck(r) + if sessionID != testSessionID { + t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) + } + if !strings.HasPrefix(sessionID, spec.OrgID) { + t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) + } + if overrides.ResponseCode != 0 { + t.Fatalf("response code should be 0, got %d", overrides.ResponseCode) + } + }) + + t.Run("BodySource", func(t *testing.T) { + extractor, spec := prepareExtractor(t, apidef.BodySource, apidef.RegexExtractor, map[string]interface{}{ + "regex_expression": extractorRegexExpr, + "regex_match_index": extractorRegexMatchIndex, + }) + + r := prepareBodyExtractorRequest("") + sessionID, overrides := extractor.ExtractAndCheck(r) + if sessionID != "" { + t.Fatalf("should return an empty session ID, got %s", sessionID) + } + if overrides.ResponseCode != 400 { + t.Fatalf("should return 400, got %d", overrides.ResponseCode) + } + + r = prepareBodyExtractorRequest(extractorRegexInput) + sessionID, overrides = extractor.ExtractAndCheck(r) + if sessionID != testSessionID { + t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) + } + if !strings.HasPrefix(sessionID, spec.OrgID) { + t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) + } + if overrides.ResponseCode != 0 { + t.Fatalf("response code should be 0, got %d", overrides.ResponseCode) + } + }) + + t.Run("FormSource", func(t *testing.T) { + extractor, spec := prepareExtractor(t, apidef.FormSource, apidef.RegexExtractor, map[string]interface{}{ + "regex_expression": extractorRegexExpr, + "regex_match_index": extractorRegexMatchIndex, + "param_name": extractorParamName, + }) + + r := prepareExtractorFormRequest(nil) + sessionID, overrides := extractor.ExtractAndCheck(r) + if sessionID != "" { + t.Fatalf("should return an empty session ID, got %s", sessionID) + } + if overrides.ResponseCode != 400 { + t.Fatalf("should return 400, got %d", overrides.ResponseCode) + } + + r = prepareExtractorFormRequest(map[string]string{extractorParamName: extractorRegexInput}) + sessionID, overrides = extractor.ExtractAndCheck(r) + if sessionID != testSessionID { + t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) + } + if !strings.HasPrefix(sessionID, spec.OrgID) { + t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) + } + if overrides.ResponseCode != 0 { + t.Fatalf("response code should be 0, got %d", overrides.ResponseCode) + } + }) +} + +func TestXPathExtractor(t *testing.T) { + testSessionID := generateSessionID("thevalue") + + t.Run("HeaderSource", func(t *testing.T) { + extractor, spec := prepareExtractor(t, apidef.HeaderSource, apidef.XPathExtractor, map[string]interface{}{ + "xpath_expression": extractorXPathExpr, + "header_name": extractorHeaderName, + }) + + r := prepareHeaderExtractorRequest(nil) + sessionID, overrides := extractor.ExtractAndCheck(r) + if sessionID != "" { + t.Fatalf("should return an empty session ID, got %s", sessionID) + } + if overrides.ResponseCode != 400 { + t.Fatalf("should return 400, got %d", overrides.ResponseCode) + } + + r = prepareHeaderExtractorRequest(map[string]string{extractorHeaderName: extractorXPathInput}) + sessionID, overrides = extractor.ExtractAndCheck(r) + if sessionID != testSessionID { + t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) + } + if !strings.HasPrefix(sessionID, spec.OrgID) { + t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) + } + if overrides.ResponseCode != 0 { + t.Fatalf("response code should be 0, got %d", overrides.ResponseCode) + } + }) + + t.Run("BodySource", func(t *testing.T) { + extractor, spec := prepareExtractor(t, apidef.BodySource, apidef.XPathExtractor, map[string]interface{}{ + "xpath_expression": extractorXPathExpr, + }) + + r := prepareBodyExtractorRequest("") + sessionID, overrides := extractor.ExtractAndCheck(r) + if sessionID != "" { + t.Fatalf("should return an empty session ID, got %s", sessionID) + } + if overrides.ResponseCode != 400 { + t.Fatalf("should return 400, got %d", overrides.ResponseCode) + } + + r = prepareBodyExtractorRequest(extractorXPathInput) + sessionID, overrides = extractor.ExtractAndCheck(r) + if sessionID != testSessionID { + t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) + } + if !strings.HasPrefix(sessionID, spec.OrgID) { + t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) + } + if overrides.ResponseCode != 0 { + t.Fatalf("response code should be 0, got %d", overrides.ResponseCode) + } + }) + + t.Run("FormSource", func(t *testing.T) { + extractor, spec := prepareExtractor(t, apidef.FormSource, apidef.XPathExtractor, map[string]interface{}{ + "xpath_expression": extractorXPathExpr, + "param_name": extractorParamName, + }) + + r := prepareExtractorFormRequest(nil) + sessionID, overrides := extractor.ExtractAndCheck(r) + if sessionID != "" { + t.Fatalf("should return an empty session ID, got %s", sessionID) + } + if overrides.ResponseCode != 400 { + t.Fatalf("should return 400, got %d", overrides.ResponseCode) + } + + r = prepareExtractorFormRequest(map[string]string{extractorParamName: extractorXPathInput}) + sessionID, overrides = extractor.ExtractAndCheck(r) + if sessionID != testSessionID { + t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) + } + if !strings.HasPrefix(sessionID, spec.OrgID) { + t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) + } + if overrides.ResponseCode != 0 { + t.Fatalf("response code should be 0, got %d", overrides.ResponseCode) + } + }) +} + +func BenchmarkValueExtractor(b *testing.B) { + b.ReportAllocs() + b.Run("HeaderSource", func(b *testing.B) { + extractor, _ := prepareExtractor(b, apidef.HeaderSource, apidef.ValueExtractor, map[string]interface{}{ + "header_name": extractorHeaderName, + }) + headers := map[string]string{extractorHeaderName: extractorValueInput} + for i := 0; i < b.N; i++ { + r := prepareHeaderExtractorRequest(headers) + extractor.ExtractAndCheck(r) + } + }) + b.Run("FormSource", func(b *testing.B) { + extractor, _ := prepareExtractor(b, apidef.FormSource, apidef.ValueExtractor, map[string]interface{}{ + "param_name": extractorParamName, + }) + params := map[string]string{extractorParamName: extractorValueInput} + for i := 0; i < b.N; i++ { + r := prepareExtractorFormRequest(params) + extractor.ExtractAndCheck(r) + } + }) +} + +func BenchmarkRegexExtractor(b *testing.B) { + b.ReportAllocs() + b.Run("HeaderSource", func(b *testing.B) { + headerName := "testheader" + extractor, _ := prepareExtractor(b, apidef.HeaderSource, apidef.RegexExtractor, map[string]interface{}{ + "regex_expression": extractorRegexExpr, + "regex_match_index": extractorRegexMatchIndex, + "header_name": headerName, + }) + headers := map[string]string{extractorHeaderName: extractorRegexInput} + for i := 0; i < b.N; i++ { + r := prepareHeaderExtractorRequest(headers) + extractor.ExtractAndCheck(r) + } + }) + b.Run("BodySource", func(b *testing.B) { + extractor, _ := prepareExtractor(b, apidef.BodySource, apidef.RegexExtractor, map[string]interface{}{ + "regex_expression": extractorRegexExpr, + "regex_match_index": extractorRegexMatchIndex, + }) + for i := 0; i < b.N; i++ { + r := prepareBodyExtractorRequest(extractorRegexInput) + extractor.ExtractAndCheck(r) + } + }) + b.Run("FormSource", func(b *testing.B) { + extractor, _ := prepareExtractor(b, apidef.FormSource, apidef.RegexExtractor, map[string]interface{}{ + "regex_expression": extractorRegexExpr, + "regex_match_index": extractorRegexMatchIndex, + "param_name": extractorParamName, + }) + params := map[string]string{extractorParamName: extractorRegexInput} + for i := 0; i < b.N; i++ { + r := prepareExtractorFormRequest(params) + extractor.ExtractAndCheck(r) + } + }) +} + +func BenchmarkXPathExtractor(b *testing.B) { + b.ReportAllocs() + b.Run("HeaderSource", func(b *testing.B) { + extractor, _ := prepareExtractor(b, apidef.HeaderSource, apidef.XPathExtractor, map[string]interface{}{ + "xpath_expression": extractorXPathExpr, + "header_name": extractorHeaderName, + }) + headers := map[string]string{extractorHeaderName: extractorXPathInput} + for i := 0; i < b.N; i++ { + r := prepareHeaderExtractorRequest(headers) + extractor.ExtractAndCheck(r) + } + }) + b.Run("BodySource", func(b *testing.B) { + extractor, _ := prepareExtractor(b, apidef.BodySource, apidef.XPathExtractor, map[string]interface{}{ + "xpath_expression": extractorXPathExpr, + }) + for i := 0; i < b.N; i++ { + r := prepareBodyExtractorRequest(extractorXPathInput) + extractor.ExtractAndCheck(r) + } + }) + b.Run("FormSource", func(b *testing.B) { + extractor, _ := prepareExtractor(b, apidef.FormSource, apidef.XPathExtractor, map[string]interface{}{ + "xpath_expression": extractorXPathExpr, + "param_name": extractorParamName, + }) + params := map[string]string{extractorParamName: extractorXPathInput} + for i := 0; i < b.N; i++ { + r := prepareExtractorFormRequest(params) + extractor.ExtractAndCheck(r) + } }) } From 0a42aa88e768dc6c5183af875a3c1e6d580c07e8 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Sun, 29 Apr 2018 23:14:12 +0200 Subject: [PATCH 02/54] id_extractor: use a precompiled regular expression when using RegexExtractor --- coprocess_id_extractor.go | 55 +++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/coprocess_id_extractor.go b/coprocess_id_extractor.go index 20408caa9cf..eb830faedf6 100644 --- a/coprocess_id_extractor.go +++ b/coprocess_id_extractor.go @@ -77,11 +77,11 @@ func (e *BaseExtractor) ExtractForm(r *http.Request, paramName string) (formValu } // ExtractBody is used when BodySource is specified. -func (e *BaseExtractor) ExtractBody(r *http.Request) (bodyValue string, err error) { +func (e *BaseExtractor) ExtractBody(r *http.Request) (string, error) { copiedRequest := copyRequest(r) body, err := ioutil.ReadAll(copiedRequest.Body) if err != nil { - return bodyValue, err + return "", err } return string(body), err } @@ -159,6 +159,8 @@ func (e *ValueExtractor) ExtractAndCheck(r *http.Request) (sessionID string, ret type RegexExtractor struct { BaseExtractor + compiledExpr *regexp.Regexp + cfg *RegexExtractorConfig } type RegexExtractorConfig struct { @@ -169,13 +171,14 @@ type RegexExtractorConfig struct { } func (e *RegexExtractor) ExtractAndCheck(r *http.Request) (SessionID string, returnOverrides ReturnOverrides) { - var extractorOutput string - var config RegexExtractorConfig - - err := mapstructure.Decode(e.Config.ExtractorConfig, &config) - if err != nil { - returnOverrides = e.Error(r, err, "Can't decode RegexExtractor configuration") - return SessionID, returnOverrides + // Parse specific configuration settings: + if e.cfg == nil { + config := &RegexExtractorConfig{} + if err := mapstructure.Decode(e.Config.ExtractorConfig, config); err != nil { + returnOverrides = e.Error(r, err, "Can't decode RegexExtractor configuration") + return SessionID, returnOverrides + } + e.cfg = config } if e.Config.ExtractorConfig["regex_expression"] == nil { @@ -183,39 +186,39 @@ func (e *RegexExtractor) ExtractAndCheck(r *http.Request) (SessionID string, ret return SessionID, returnOverrides } - expression, err := regexp.Compile(config.RegexExpression) - - if err != nil { - returnOverrides = e.Error(r, nil, "RegexExtractor found an invalid expression") - return SessionID, returnOverrides + var err error + if e.compiledExpr == nil { + e.compiledExpr, err = regexp.Compile(e.cfg.RegexExpression) + if err != nil { + returnOverrides = e.Error(r, nil, "RegexExtractor found an invalid expression") + return SessionID, returnOverrides + } } + var extractorOutput string switch e.Config.ExtractFrom { case apidef.HeaderSource: extractorOutput, err = e.ExtractHeader(r) case apidef.BodySource: extractorOutput, err = e.ExtractBody(r) case apidef.FormSource: - extractorOutput, err = e.ExtractForm(r, config.FormParamName) + extractorOutput, err = e.ExtractForm(r, e.cfg.FormParamName) } - if err != nil { returnOverrides = e.Error(r, err, "RegexExtractor error") return SessionID, returnOverrides } - regexOutput := expression.FindAllString(extractorOutput, -1) - if config.RegexMatchIndex > len(regexOutput)-1 { + regexOutput := e.compiledExpr.FindAllString(extractorOutput, -1) + if e.cfg.RegexMatchIndex > len(regexOutput)-1 { returnOverrides = e.Error(r, fmt.Errorf("Can't find regexp match group"), "RegexExtractor error") return SessionID, returnOverrides } - SessionID = e.GenerateSessionID(regexOutput[config.RegexMatchIndex], e.BaseMid) - + SessionID = e.GenerateSessionID(regexOutput[e.cfg.RegexMatchIndex], e.BaseMid) previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(SessionID) if keyExists { - if previousSession.IdExtractorDeadline > time.Now().Unix() { e.PostProcess(r, &previousSession, SessionID) returnOverrides = ReturnOverrides{ @@ -223,7 +226,6 @@ func (e *RegexExtractor) ExtractAndCheck(r *http.Request) (SessionID string, ret } } } - return SessionID, returnOverrides } @@ -232,8 +234,10 @@ type XPathExtractor struct { } type XPathExtractorConfig struct { - HeaderName string `mapstructure:"header_name" bson:"header_name" json:"header_name"` - FormParamName string `mapstructure:"param_name" bson:"param_name" json:"param_name"` + HeaderName string `mapstructure:"header_name" bson:"header_name" json:"header_name"` + RegexExpression string `mapstructure:"regex_expression" bson:"regex_expression" json:"regex_expression"` + RegexMatchIndex int `mapstructure:"regex_match_index" bson:"regex_match_index" json:"regex_match_index"` + FormParamName string `mapstructure:"param_name" bson:"param_name" json:"param_name"` } func (e *XPathExtractor) ExtractAndCheck(r *http.Request) (SessionID string, returnOverrides ReturnOverrides) { @@ -248,6 +252,7 @@ func (e *XPathExtractor) ExtractAndCheck(r *http.Request) (SessionID string, ret } expressionString := e.Config.ExtractorConfig["xpath_expression"].(string) + expression, err := xmlpath.Compile(expressionString) if err != nil { returnOverrides = e.Error(r, err, "XPathExtractor: bad expression") @@ -305,7 +310,7 @@ func newExtractor(referenceSpec *APISpec, mw BaseMiddleware) { case apidef.ValueExtractor: extractor = &ValueExtractor{baseExtractor} case apidef.RegexExtractor: - extractor = &RegexExtractor{baseExtractor} + extractor = &RegexExtractor{baseExtractor, nil, nil} case apidef.XPathExtractor: extractor = &XPathExtractor{baseExtractor} } From 5e4cdf2a72e911c7c98c287e6839f111aed27827 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Thu, 17 May 2018 20:37:43 -0400 Subject: [PATCH 03/54] Add ReportAllocs call to sub-benchmarks --- api_definition_test.go | 3 +++ coprocess_id_extractor_test.go | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/api_definition_test.go b/api_definition_test.go index 7af66b979bc..0ac08b39b6e 100644 --- a/api_definition_test.go +++ b/api_definition_test.go @@ -459,6 +459,7 @@ func BenchmarkGetVersionFromRequest(b *testing.B) { versionInfo.Paths.BlackList = []string{"/bar"} b.Run("Header location", func(b *testing.B) { + b.ReportAllocs() buildAndLoadAPI(func(spec *APISpec) { spec.Proxy.ListenPath = "/" spec.VersionData.NotVersioned = false @@ -476,6 +477,7 @@ func BenchmarkGetVersionFromRequest(b *testing.B) { }) b.Run("URL param location", func(b *testing.B) { + b.ReportAllocs() buildAndLoadAPI(func(spec *APISpec) { spec.Proxy.ListenPath = "/" spec.VersionData.NotVersioned = false @@ -493,6 +495,7 @@ func BenchmarkGetVersionFromRequest(b *testing.B) { }) b.Run("URL location", func(b *testing.B) { + b.ReportAllocs() buildAndLoadAPI(func(spec *APISpec) { spec.Proxy.ListenPath = "/" spec.VersionData.NotVersioned = false diff --git a/coprocess_id_extractor_test.go b/coprocess_id_extractor_test.go index e6410acd056..707861d208c 100644 --- a/coprocess_id_extractor_test.go +++ b/coprocess_id_extractor_test.go @@ -326,8 +326,8 @@ func TestXPathExtractor(t *testing.T) { } func BenchmarkValueExtractor(b *testing.B) { - b.ReportAllocs() b.Run("HeaderSource", func(b *testing.B) { + b.ReportAllocs() extractor, _ := prepareExtractor(b, apidef.HeaderSource, apidef.ValueExtractor, map[string]interface{}{ "header_name": extractorHeaderName, }) @@ -338,6 +338,7 @@ func BenchmarkValueExtractor(b *testing.B) { } }) b.Run("FormSource", func(b *testing.B) { + b.ReportAllocs() extractor, _ := prepareExtractor(b, apidef.FormSource, apidef.ValueExtractor, map[string]interface{}{ "param_name": extractorParamName, }) @@ -350,8 +351,8 @@ func BenchmarkValueExtractor(b *testing.B) { } func BenchmarkRegexExtractor(b *testing.B) { - b.ReportAllocs() b.Run("HeaderSource", func(b *testing.B) { + b.ReportAllocs() headerName := "testheader" extractor, _ := prepareExtractor(b, apidef.HeaderSource, apidef.RegexExtractor, map[string]interface{}{ "regex_expression": extractorRegexExpr, @@ -365,6 +366,7 @@ func BenchmarkRegexExtractor(b *testing.B) { } }) b.Run("BodySource", func(b *testing.B) { + b.ReportAllocs() extractor, _ := prepareExtractor(b, apidef.BodySource, apidef.RegexExtractor, map[string]interface{}{ "regex_expression": extractorRegexExpr, "regex_match_index": extractorRegexMatchIndex, @@ -375,6 +377,7 @@ func BenchmarkRegexExtractor(b *testing.B) { } }) b.Run("FormSource", func(b *testing.B) { + b.ReportAllocs() extractor, _ := prepareExtractor(b, apidef.FormSource, apidef.RegexExtractor, map[string]interface{}{ "regex_expression": extractorRegexExpr, "regex_match_index": extractorRegexMatchIndex, @@ -389,8 +392,8 @@ func BenchmarkRegexExtractor(b *testing.B) { } func BenchmarkXPathExtractor(b *testing.B) { - b.ReportAllocs() b.Run("HeaderSource", func(b *testing.B) { + b.ReportAllocs() extractor, _ := prepareExtractor(b, apidef.HeaderSource, apidef.XPathExtractor, map[string]interface{}{ "xpath_expression": extractorXPathExpr, "header_name": extractorHeaderName, @@ -402,6 +405,7 @@ func BenchmarkXPathExtractor(b *testing.B) { } }) b.Run("BodySource", func(b *testing.B) { + b.ReportAllocs() extractor, _ := prepareExtractor(b, apidef.BodySource, apidef.XPathExtractor, map[string]interface{}{ "xpath_expression": extractorXPathExpr, }) @@ -411,6 +415,7 @@ func BenchmarkXPathExtractor(b *testing.B) { } }) b.Run("FormSource", func(b *testing.B) { + b.ReportAllocs() extractor, _ := prepareExtractor(b, apidef.FormSource, apidef.XPathExtractor, map[string]interface{}{ "xpath_expression": extractorXPathExpr, "param_name": extractorParamName, From 39387a99fee5a8157daa55a9f62e578ad5358e4b Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Tue, 1 May 2018 16:35:26 +0200 Subject: [PATCH 04/54] id_extractor: use precompiled XPath expression when using XPathExtractor Modifies ValueExtractor and XPathExtractor to avoid decoding configuration multiple times --- coprocess_id_extractor.go | 55 +++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/coprocess_id_extractor.go b/coprocess_id_extractor.go index eb830faedf6..7be112d878b 100644 --- a/coprocess_id_extractor.go +++ b/coprocess_id_extractor.go @@ -107,6 +107,7 @@ func (e *BaseExtractor) GenerateSessionID(input string, mw BaseMiddleware) (sess type ValueExtractor struct { BaseExtractor + cfg *ValueExtractorConfig } type ValueExtractorConfig struct { @@ -120,20 +121,22 @@ func (e *ValueExtractor) Extract(input interface{}) string { } func (e *ValueExtractor) ExtractAndCheck(r *http.Request) (sessionID string, returnOverrides ReturnOverrides) { - var extractorOutput string - var config ValueExtractorConfig - - err := mapstructure.Decode(e.Config.ExtractorConfig, &config) - if err != nil { - returnOverrides = e.Error(r, err, "Couldn't decode ValueExtractor configuration") - return sessionID, returnOverrides + if e.cfg == nil { + config := &ValueExtractorConfig{} + if err := mapstructure.Decode(e.Config.ExtractorConfig, config); err != nil { + returnOverrides = e.Error(r, err, "Couldn't decode ValueExtractor configuration") + return sessionID, returnOverrides + } + e.cfg = config } + var extractorOutput string + var err error switch e.Config.ExtractFrom { case apidef.HeaderSource: extractorOutput, err = e.ExtractHeader(r) case apidef.FormSource: - extractorOutput, err = e.ExtractForm(r, config.FormParamName) + extractorOutput, err = e.ExtractForm(r, e.cfg.FormParamName) } if err != nil { @@ -231,6 +234,8 @@ func (e *RegexExtractor) ExtractAndCheck(r *http.Request) (SessionID string, ret type XPathExtractor struct { BaseExtractor + cfg *XPathExtractorConfig + path *xmlpath.Path } type XPathExtractorConfig struct { @@ -241,31 +246,35 @@ type XPathExtractorConfig struct { } func (e *XPathExtractor) ExtractAndCheck(r *http.Request) (SessionID string, returnOverrides ReturnOverrides) { - var extractorOutput string - - var config XPathExtractorConfig - err := mapstructure.Decode(e.Config.ExtractorConfig, &config) - + var err error + config := &XPathExtractorConfig{} + if err = mapstructure.Decode(e.Config.ExtractorConfig, config); err != nil { + returnOverrides = e.Error(r, err, "Can't decode XPathExtractor configuration") + return SessionID, returnOverrides + } + e.cfg = config if e.Config.ExtractorConfig["xpath_expression"] == nil { returnOverrides = e.Error(r, err, "XPathExtractor: no expression set") return SessionID, returnOverrides } - expressionString := e.Config.ExtractorConfig["xpath_expression"].(string) - - expression, err := xmlpath.Compile(expressionString) - if err != nil { - returnOverrides = e.Error(r, err, "XPathExtractor: bad expression") - return SessionID, returnOverrides + if e.path == nil { + expressionString := e.Config.ExtractorConfig["xpath_expression"].(string) + e.path, err = xmlpath.Compile(expressionString) + if err != nil { + returnOverrides = e.Error(r, err, "XPathExtractor: bad expression") + return SessionID, returnOverrides + } } + var extractorOutput string switch e.Config.ExtractFrom { case apidef.HeaderSource: extractorOutput, err = e.ExtractHeader(r) case apidef.BodySource: extractorOutput, err = e.ExtractBody(r) case apidef.FormSource: - extractorOutput, err = e.ExtractForm(r, config.FormParamName) + extractorOutput, err = e.ExtractForm(r, e.cfg.FormParamName) } if err != nil { returnOverrides = e.Error(r, err, "XPathExtractor error") @@ -278,7 +287,7 @@ func (e *XPathExtractor) ExtractAndCheck(r *http.Request) (SessionID string, ret return SessionID, returnOverrides } - output, ok := expression.String(extractedXml) + output, ok := e.path.String(extractedXml) if !ok { returnOverrides = e.Error(r, err, "XPathExtractor: no input") return SessionID, returnOverrides @@ -308,11 +317,11 @@ func newExtractor(referenceSpec *APISpec, mw BaseMiddleware) { // Initialize a extractor based on the API spec. switch referenceSpec.CustomMiddleware.IdExtractor.ExtractWith { case apidef.ValueExtractor: - extractor = &ValueExtractor{baseExtractor} + extractor = &ValueExtractor{baseExtractor, nil} case apidef.RegexExtractor: extractor = &RegexExtractor{baseExtractor, nil, nil} case apidef.XPathExtractor: - extractor = &XPathExtractor{baseExtractor} + extractor = &XPathExtractor{baseExtractor, nil, nil} } referenceSpec.CustomMiddleware.IdExtractor.Extractor = extractor From bc5d9314ddca77f4d08f0065002c51fa5fa2f39e Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 18 May 2018 21:15:23 +0300 Subject: [PATCH 05/54] Fix pinned domain issue when it contain dots --- apidef/api_definitions.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/apidef/api_definitions.go b/apidef/api_definitions.go index 0e097642055..ae73a02e647 100644 --- a/apidef/api_definitions.go +++ b/apidef/api_definitions.go @@ -457,6 +457,13 @@ func (a *APIDefinition) EncodeForDB() { } a.UpstreamCertificates = newUpstreamCerts + newPinnedPublicKeys := make(map[string]string) + for domain, cert := range a.PinnedPublicKeys { + newD := base64.StdEncoding.EncodeToString([]byte(domain)) + newPinnedPublicKeys[newD] = cert + } + a.PinnedPublicKeys = newPinnedPublicKeys + for i, version := range a.VersionData.Versions { for j, oldSchema := range version.ExtendedPaths.ValidateJSON { @@ -495,6 +502,18 @@ func (a *APIDefinition) DecodeFromDB() { } a.UpstreamCertificates = newUpstreamCerts + newPinnedPublicKeys := make(map[string]string) + for domain, cert := range a.PinnedPublicKeys { + newD, err := base64.StdEncoding.DecodeString(domain) + if err != nil { + log.Error("Couldn't Decode, leaving as it may be legacy...") + newPinnedPublicKeys[domain] = cert + } else { + newPinnedPublicKeys[string(newD)] = cert + } + } + a.PinnedPublicKeys = newPinnedPublicKeys + for i, version := range a.VersionData.Versions { for j, oldSchema := range version.ExtendedPaths.ValidateJSON { jsBytes, _ := base64.StdEncoding.DecodeString(oldSchema.SchemaB64) From afde0327997498b46fb532045c2cc80d320d1ef8 Mon Sep 17 00:00:00 2001 From: joshblakeley Date: Wed, 16 May 2018 14:19:32 +0100 Subject: [PATCH 06/54] expose request protocol in request object in jsvm --- mw_js_plugin.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mw_js_plugin.go b/mw_js_plugin.go index 6cb20c9371e..d9e5196e50c 100644 --- a/mw_js_plugin.go +++ b/mw_js_plugin.go @@ -45,6 +45,7 @@ type MiniRequestObject struct { IgnoreBody bool Method string RequestURI string + Proto string } type VMReturnObject struct { @@ -96,7 +97,6 @@ func (d *DynamicMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Reques }).Error("Failed to read request body! ", err) return nil, http.StatusOK } - headers := r.Header host := r.Host if host == "" && r.URL != nil { @@ -122,6 +122,7 @@ func (d *DynamicMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Reques DeleteParams: []string{}, Method: r.Method, RequestURI: r.RequestURI, + Proto: r.Proto, } requestAsJson, err := json.Marshal(requestData) From 12968500ecec27fe99d03907de9c5598aaf66499 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Wed, 16 May 2018 18:45:37 -0400 Subject: [PATCH 07/54] coprocess: expose request protocol (#1702) Update bindings using "update_bindings.sh". --- coprocess.go | 1 + .../coprocess_mini_request_object_pb2.py | 31 +++++---- .../ruby/coprocess_mini_request_object_pb.rb | 1 + coprocess/coprocess_mini_request_object.pb.go | 64 +++++++++++-------- .../proto/coprocess_mini_request_object.proto | 1 + 5 files changed, 58 insertions(+), 40 deletions(-) diff --git a/coprocess.go b/coprocess.go index b59795711b7..0f2996a3fa5 100644 --- a/coprocess.go +++ b/coprocess.go @@ -101,6 +101,7 @@ func (c *CoProcessor) ObjectFromRequest(r *http.Request) *coprocess.Object { }, Method: r.Method, RequestUri: r.RequestURI, + Proto: r.Proto, } object := &coprocess.Object{ diff --git a/coprocess/bindings/python/coprocess_mini_request_object_pb2.py b/coprocess/bindings/python/coprocess_mini_request_object_pb2.py index cac4f4caaac..d604f096c32 100644 --- a/coprocess/bindings/python/coprocess_mini_request_object_pb2.py +++ b/coprocess/bindings/python/coprocess_mini_request_object_pb2.py @@ -20,7 +20,7 @@ name='coprocess_mini_request_object.proto', package='coprocess', syntax='proto3', - serialized_pb=_b('\n#coprocess_mini_request_object.proto\x12\tcoprocess\x1a coprocess_return_overrides.proto\"\xf8\x05\n\x11MiniRequestObject\x12:\n\x07headers\x18\x01 \x03(\x0b\x32).coprocess.MiniRequestObject.HeadersEntry\x12\x41\n\x0bset_headers\x18\x02 \x03(\x0b\x32,.coprocess.MiniRequestObject.SetHeadersEntry\x12\x16\n\x0e\x64\x65lete_headers\x18\x03 \x03(\t\x12\x0c\n\x04\x62ody\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x38\n\x06params\x18\x06 \x03(\x0b\x32(.coprocess.MiniRequestObject.ParamsEntry\x12?\n\nadd_params\x18\x07 \x03(\x0b\x32+.coprocess.MiniRequestObject.AddParamsEntry\x12I\n\x0f\x65xtended_params\x18\x08 \x03(\x0b\x32\x30.coprocess.MiniRequestObject.ExtendedParamsEntry\x12\x15\n\rdelete_params\x18\t \x03(\t\x12\x34\n\x10return_overrides\x18\n \x01(\x0b\x32\x1a.coprocess.ReturnOverrides\x12\x0e\n\x06method\x18\x0b \x01(\t\x12\x13\n\x0brequest_uri\x18\x0c \x01(\t\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fSetHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a-\n\x0bParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x30\n\x0e\x41\x64\x64ParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x35\n\x13\x45xtendedParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3') + serialized_pb=_b('\n#coprocess_mini_request_object.proto\x12\tcoprocess\x1a coprocess_return_overrides.proto\"\x87\x06\n\x11MiniRequestObject\x12:\n\x07headers\x18\x01 \x03(\x0b\x32).coprocess.MiniRequestObject.HeadersEntry\x12\x41\n\x0bset_headers\x18\x02 \x03(\x0b\x32,.coprocess.MiniRequestObject.SetHeadersEntry\x12\x16\n\x0e\x64\x65lete_headers\x18\x03 \x03(\t\x12\x0c\n\x04\x62ody\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x38\n\x06params\x18\x06 \x03(\x0b\x32(.coprocess.MiniRequestObject.ParamsEntry\x12?\n\nadd_params\x18\x07 \x03(\x0b\x32+.coprocess.MiniRequestObject.AddParamsEntry\x12I\n\x0f\x65xtended_params\x18\x08 \x03(\x0b\x32\x30.coprocess.MiniRequestObject.ExtendedParamsEntry\x12\x15\n\rdelete_params\x18\t \x03(\t\x12\x34\n\x10return_overrides\x18\n \x01(\x0b\x32\x1a.coprocess.ReturnOverrides\x12\x0e\n\x06method\x18\x0b \x01(\t\x12\x13\n\x0brequest_uri\x18\x0c \x01(\t\x12\r\n\x05proto\x18\r \x01(\t\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fSetHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a-\n\x0bParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x30\n\x0e\x41\x64\x64ParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x35\n\x13\x45xtendedParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3') , dependencies=[coprocess__return__overrides__pb2.DESCRIPTOR,]) @@ -60,8 +60,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=596, - serialized_end=642, + serialized_start=611, + serialized_end=657, ) _MINIREQUESTOBJECT_SETHEADERSENTRY = _descriptor.Descriptor( @@ -97,8 +97,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=644, - serialized_end=693, + serialized_start=659, + serialized_end=708, ) _MINIREQUESTOBJECT_PARAMSENTRY = _descriptor.Descriptor( @@ -134,8 +134,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=695, - serialized_end=740, + serialized_start=710, + serialized_end=755, ) _MINIREQUESTOBJECT_ADDPARAMSENTRY = _descriptor.Descriptor( @@ -171,8 +171,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=742, - serialized_end=790, + serialized_start=757, + serialized_end=805, ) _MINIREQUESTOBJECT_EXTENDEDPARAMSENTRY = _descriptor.Descriptor( @@ -208,8 +208,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=792, - serialized_end=845, + serialized_start=807, + serialized_end=860, ) _MINIREQUESTOBJECT = _descriptor.Descriptor( @@ -303,6 +303,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='proto', full_name='coprocess.MiniRequestObject.proto', index=12, + number=13, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -316,7 +323,7 @@ oneofs=[ ], serialized_start=85, - serialized_end=845, + serialized_end=860, ) _MINIREQUESTOBJECT_HEADERSENTRY.containing_type = _MINIREQUESTOBJECT diff --git a/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb b/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb index 757c210e889..2f479ea0814 100644 --- a/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb +++ b/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb @@ -18,6 +18,7 @@ optional :return_overrides, :message, 10, "coprocess.ReturnOverrides" optional :method, :string, 11 optional :request_uri, :string, 12 + optional :proto, :string, 13 end end diff --git a/coprocess/coprocess_mini_request_object.pb.go b/coprocess/coprocess_mini_request_object.pb.go index 0704b66c439..f8deb7c0383 100644 --- a/coprocess/coprocess_mini_request_object.pb.go +++ b/coprocess/coprocess_mini_request_object.pb.go @@ -25,6 +25,7 @@ type MiniRequestObject struct { ReturnOverrides *ReturnOverrides `protobuf:"bytes,10,opt,name=return_overrides,json=returnOverrides" json:"return_overrides,omitempty"` Method string `protobuf:"bytes,11,opt,name=method" json:"method,omitempty"` RequestUri string `protobuf:"bytes,12,opt,name=request_uri,json=requestUri" json:"request_uri,omitempty"` + Proto string `protobuf:"bytes,13,opt,name=proto" json:"proto,omitempty"` } func (m *MiniRequestObject) Reset() { *m = MiniRequestObject{} } @@ -116,6 +117,13 @@ func (m *MiniRequestObject) GetRequestUri() string { return "" } +func (m *MiniRequestObject) GetProto() string { + if m != nil { + return m.Proto + } + return "" +} + func init() { proto.RegisterType((*MiniRequestObject)(nil), "coprocess.MiniRequestObject") } @@ -123,32 +131,32 @@ func init() { func init() { proto.RegisterFile("coprocess_mini_request_object.proto", fileDescriptor1) } var fileDescriptor1 = []byte{ - // 420 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0xd1, 0x8b, 0xd3, 0x40, - 0x10, 0xc6, 0x49, 0x73, 0x97, 0x9a, 0x49, 0x6d, 0xcf, 0x55, 0x64, 0xc9, 0x8b, 0xc1, 0x43, 0x88, - 0x28, 0x41, 0xce, 0x17, 0x3d, 0x14, 0x3c, 0xa4, 0x20, 0x42, 0xa9, 0x44, 0x7c, 0xf0, 0x29, 0xa4, - 0xdd, 0x81, 0xc6, 0xb6, 0xd9, 0xba, 0xbb, 0x29, 0xe6, 0x4f, 0xf3, 0xbf, 0x93, 0xee, 0x26, 0x31, - 0xa9, 0x12, 0xc8, 0xdb, 0xce, 0x97, 0xef, 0xfb, 0x31, 0xcc, 0x4c, 0xe0, 0x7a, 0xcd, 0x0f, 0x82, - 0xaf, 0x51, 0xca, 0x64, 0x9f, 0xe5, 0x59, 0x22, 0xf0, 0x67, 0x81, 0x52, 0x25, 0x7c, 0xf5, 0x03, - 0xd7, 0x2a, 0x3a, 0x08, 0xae, 0x38, 0x71, 0x1b, 0x93, 0x1f, 0xfc, 0xf5, 0x0b, 0x54, 0x85, 0xc8, - 0x13, 0x7e, 0x44, 0x21, 0x32, 0x86, 0xd2, 0x98, 0x9f, 0xfe, 0x1e, 0xc3, 0x83, 0x45, 0x96, 0x67, - 0xb1, 0x21, 0x2d, 0x35, 0x88, 0x7c, 0x84, 0xf1, 0x06, 0x53, 0x86, 0x42, 0x52, 0x2b, 0xb0, 0x43, - 0xef, 0xe6, 0x79, 0xd4, 0x90, 0xa2, 0x7f, 0xec, 0xd1, 0x27, 0xe3, 0x9d, 0xe7, 0x4a, 0x94, 0x71, - 0x9d, 0x24, 0x0b, 0xf0, 0x24, 0xaa, 0xa4, 0x06, 0x8d, 0x34, 0xe8, 0x65, 0x2f, 0xe8, 0x2b, 0xaa, - 0x0e, 0x0b, 0x64, 0x23, 0x90, 0x67, 0x30, 0x65, 0xb8, 0x43, 0x85, 0x0d, 0xd1, 0x0e, 0xec, 0xd0, - 0x8d, 0xef, 0x1b, 0xb5, 0xb6, 0x11, 0xb8, 0x58, 0x71, 0x56, 0xd2, 0x8b, 0xc0, 0x0a, 0xdd, 0x58, - 0xbf, 0xc9, 0x15, 0xd8, 0x85, 0xd8, 0xd1, 0x4b, 0x2d, 0x9d, 0x9e, 0xe4, 0x03, 0x38, 0x87, 0x54, - 0xa4, 0x7b, 0x49, 0x1d, 0xdd, 0x56, 0xd8, 0xdb, 0xd6, 0x17, 0x6d, 0x35, 0x2d, 0x55, 0x39, 0xf2, - 0x19, 0x20, 0x65, 0x2c, 0xa9, 0x28, 0x63, 0x4d, 0x79, 0xd1, 0x4b, 0xb9, 0x63, 0xac, 0x0d, 0x72, - 0xd3, 0xba, 0x26, 0xdf, 0x61, 0x86, 0xbf, 0x14, 0xe6, 0x0c, 0x1b, 0xe0, 0x3d, 0x0d, 0x7c, 0xd5, - 0x0b, 0x9c, 0x57, 0x99, 0x36, 0x75, 0x8a, 0x1d, 0x91, 0x5c, 0x43, 0x35, 0x9f, 0x1a, 0xec, 0xea, - 0xa1, 0x4d, 0x8c, 0x58, 0x99, 0xe6, 0x70, 0x75, 0x7e, 0x1e, 0x14, 0x02, 0x2b, 0xf4, 0x6e, 0xfc, - 0x56, 0x03, 0xb1, 0xb6, 0x2c, 0x6b, 0x47, 0x3c, 0x13, 0x5d, 0x81, 0x3c, 0x06, 0x67, 0x8f, 0x6a, - 0xc3, 0x19, 0xf5, 0xf4, 0xa4, 0xab, 0x8a, 0x3c, 0x01, 0xaf, 0x3e, 0xd4, 0x42, 0x64, 0x74, 0xa2, - 0x3f, 0x42, 0x25, 0x7d, 0x13, 0x99, 0x7f, 0x0b, 0x93, 0xf6, 0xda, 0x4f, 0xfb, 0xda, 0x62, 0x49, - 0x2d, 0xb3, 0xaf, 0x2d, 0x96, 0xe4, 0x11, 0x5c, 0x1e, 0xd3, 0x5d, 0x81, 0x74, 0xa4, 0x35, 0x53, - 0xdc, 0x8e, 0xde, 0x58, 0xfe, 0x7b, 0x98, 0x9d, 0x5d, 0xcd, 0xa0, 0xf8, 0x5b, 0xf0, 0x5a, 0xe3, - 0x1b, 0x14, 0x7d, 0x07, 0xd3, 0xee, 0x4a, 0x07, 0xa5, 0xef, 0xe0, 0xe1, 0x7f, 0xf6, 0x37, 0x04, - 0xb1, 0x72, 0xf4, 0x2f, 0xfc, 0xfa, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xf3, 0xdc, 0xc9, 0x33, - 0x16, 0x04, 0x00, 0x00, + // 429 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x92, 0xd1, 0x8b, 0xd3, 0x40, + 0x10, 0xc6, 0x49, 0x7b, 0xd7, 0x9a, 0x49, 0xaf, 0x3d, 0x57, 0x91, 0x25, 0x2f, 0x06, 0x0f, 0x21, + 0xa2, 0x04, 0x39, 0x5f, 0xf4, 0x50, 0xf0, 0x90, 0x82, 0x08, 0xc7, 0x49, 0xc4, 0x07, 0x9f, 0x42, + 0xda, 0x1d, 0xb8, 0x78, 0x6d, 0xb6, 0xee, 0x6e, 0x0e, 0xf3, 0x57, 0xfa, 0x2f, 0x49, 0x67, 0x93, + 0x98, 0xd4, 0x23, 0x90, 0xb7, 0x9d, 0x6f, 0xbf, 0xef, 0xc7, 0x30, 0x33, 0x70, 0xb6, 0x96, 0x3b, + 0x25, 0xd7, 0xa8, 0x75, 0xb2, 0xcd, 0xf2, 0x2c, 0x51, 0xf8, 0xab, 0x40, 0x6d, 0x12, 0xb9, 0xfa, + 0x89, 0x6b, 0x13, 0xed, 0x94, 0x34, 0x92, 0xb9, 0x8d, 0xc9, 0x0f, 0xfe, 0xf9, 0x15, 0x9a, 0x42, + 0xe5, 0x89, 0xbc, 0x43, 0xa5, 0x32, 0x81, 0xda, 0x9a, 0x9f, 0xfd, 0x99, 0xc2, 0xc3, 0xab, 0x2c, + 0xcf, 0x62, 0x4b, 0xba, 0x26, 0x10, 0xfb, 0x04, 0xd3, 0x1b, 0x4c, 0x05, 0x2a, 0xcd, 0x9d, 0x60, + 0x1c, 0x7a, 0xe7, 0x2f, 0xa2, 0x86, 0x14, 0xfd, 0x67, 0x8f, 0x3e, 0x5b, 0xef, 0x32, 0x37, 0xaa, + 0x8c, 0xeb, 0x24, 0xbb, 0x02, 0x4f, 0xa3, 0x49, 0x6a, 0xd0, 0x88, 0x40, 0xaf, 0x7a, 0x41, 0xdf, + 0xd0, 0x74, 0x58, 0xa0, 0x1b, 0x81, 0x3d, 0x87, 0xb9, 0xc0, 0x0d, 0x1a, 0x6c, 0x88, 0xe3, 0x60, + 0x1c, 0xba, 0xf1, 0x89, 0x55, 0x6b, 0x1b, 0x83, 0xa3, 0x95, 0x14, 0x25, 0x3f, 0x0a, 0x9c, 0xd0, + 0x8d, 0xe9, 0xcd, 0x4e, 0x61, 0x5c, 0xa8, 0x0d, 0x3f, 0x26, 0x69, 0xff, 0x64, 0x1f, 0x61, 0xb2, + 0x4b, 0x55, 0xba, 0xd5, 0x7c, 0x42, 0x6d, 0x85, 0xbd, 0x6d, 0x7d, 0x25, 0xab, 0x6d, 0xa9, 0xca, + 0xb1, 0x2f, 0x00, 0xa9, 0x10, 0x49, 0x45, 0x99, 0x12, 0xe5, 0x65, 0x2f, 0xe5, 0x52, 0x88, 0x36, + 0xc8, 0x4d, 0xeb, 0x9a, 0xfd, 0x80, 0x05, 0xfe, 0x36, 0x98, 0x0b, 0x6c, 0x80, 0x0f, 0x08, 0xf8, + 0xba, 0x17, 0xb8, 0xac, 0x32, 0x6d, 0xea, 0x1c, 0x3b, 0x22, 0x3b, 0x83, 0x6a, 0x3e, 0x35, 0xd8, + 0xa5, 0xa1, 0xcd, 0xac, 0x58, 0x99, 0x96, 0x70, 0x7a, 0x78, 0x1e, 0x1c, 0x02, 0x27, 0xf4, 0xce, + 0xfd, 0x56, 0x03, 0x31, 0x59, 0xae, 0x6b, 0x47, 0xbc, 0x50, 0x5d, 0x81, 0x3d, 0x81, 0xc9, 0x16, + 0xcd, 0x8d, 0x14, 0xdc, 0xa3, 0x49, 0x57, 0x15, 0x7b, 0x0a, 0x5e, 0x7d, 0xa8, 0x85, 0xca, 0xf8, + 0x8c, 0x3e, 0xa1, 0x92, 0xbe, 0xab, 0x8c, 0x3d, 0x86, 0x63, 0xba, 0x46, 0x7e, 0x42, 0x5f, 0xb6, + 0xf0, 0x2f, 0x60, 0xd6, 0x3e, 0x86, 0xfd, 0x16, 0x6f, 0xb1, 0xe4, 0x8e, 0xdd, 0xe2, 0x2d, 0x96, + 0xfb, 0xdc, 0x5d, 0xba, 0x29, 0x90, 0x8f, 0x6c, 0x8e, 0x8a, 0x8b, 0xd1, 0x5b, 0xc7, 0xff, 0x00, + 0x8b, 0x83, 0x5b, 0x1a, 0x14, 0x7f, 0x07, 0x5e, 0x6b, 0xa8, 0x83, 0xa2, 0xef, 0x61, 0xde, 0x5d, + 0xf4, 0xa0, 0xf4, 0x25, 0x3c, 0xba, 0x67, 0xab, 0x43, 0x10, 0xab, 0x09, 0x4d, 0xef, 0xcd, 0xdf, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x60, 0xc5, 0x9f, 0x29, 0x2c, 0x04, 0x00, 0x00, } diff --git a/coprocess/proto/coprocess_mini_request_object.proto b/coprocess/proto/coprocess_mini_request_object.proto index 8fee35411d2..8298fbffbc8 100644 --- a/coprocess/proto/coprocess_mini_request_object.proto +++ b/coprocess/proto/coprocess_mini_request_object.proto @@ -18,4 +18,5 @@ message MiniRequestObject { ReturnOverrides return_overrides = 10; string method = 11; string request_uri = 12; + string proto = 13; } From a425019193efd8b5eb29a13e328ccdeb354e84e2 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Sat, 19 May 2018 01:18:45 -0400 Subject: [PATCH 08/54] Update global event handlers code (#1679) * main: update generic event handlers code to use config.Global, fixes #1678. Includes patch from #1693. * Add event system tests --- event_system.go | 24 +++++-- event_system_test.go | 164 +++++++++++++++++++++++++++++++++++++++++++ main.go | 18 ++--- 3 files changed, 187 insertions(+), 19 deletions(-) create mode 100644 event_system_test.go diff --git a/event_system.go b/event_system.go index 4c07e8b810b..9ed6aac3a34 100644 --- a/event_system.go +++ b/event_system.go @@ -7,6 +7,7 @@ import ( "net/http" "time" + "github.com/Sirupsen/logrus" "github.com/rubyist/circuitbreaker" "github.com/TykTechnologies/tyk/apidef" @@ -159,11 +160,20 @@ func FireSystemEvent(name apidef.TykEvent, meta interface{}) { // LogMessageEventHandler is a sample Event Handler type LogMessageEventHandler struct { prefix string + logger *logrus.Logger } // New enables the intitialisation of event handler instances when they are created on ApiSpec creation func (l *LogMessageEventHandler) Init(handlerConf interface{}) error { - l.prefix = handlerConf.(map[string]interface{})["prefix"].(string) + conf := handlerConf.(map[string]interface{}) + l.prefix = conf["prefix"].(string) + l.logger = log + if runningTests { + logger, ok := conf["logger"] + if ok { + l.logger = logger.(*logrus.Logger) + } + } return nil } @@ -182,12 +192,12 @@ func (l *LogMessageEventHandler) HandleEvent(em config.EventMessage) { logMsg = logMsg + ":" + msgConf.APIID + ":" + msgConf.Path + ": [STATUS] " + string(msgConf.CircuitEvent) } - log.Warning(logMsg) + l.logger.Warning(logMsg) } -func InitGenericEventHandlers(theseEvents apidef.EventHandlerMetaConfig) map[apidef.TykEvent][]config.TykEventHandler { - actualEventHandlers := make(map[apidef.TykEvent][]config.TykEventHandler) - for eventName, eventHandlerConfs := range theseEvents.Events { +func initGenericEventHandlers(conf *config.Config) { + handlers := make(map[apidef.TykEvent][]config.TykEventHandler) + for eventName, eventHandlerConfs := range conf.EventHandlers.Events { log.Debug("FOUND EVENTS TO INIT") for _, handlerConf := range eventHandlerConfs { log.Debug("CREATING EVENT HANDLERS") @@ -197,10 +207,10 @@ func InitGenericEventHandlers(theseEvents apidef.EventHandlerMetaConfig) map[api log.Error("Failed to init event handler: ", err) } else { log.Debug("Init Event Handler: ", eventName) - actualEventHandlers[eventName] = append(actualEventHandlers[eventName], eventHandlerInstance) + handlers[eventName] = append(handlers[eventName], eventHandlerInstance) } } } - return actualEventHandlers + conf.EventTriggers = handlers } diff --git a/event_system_test.go b/event_system_test.go new file mode 100644 index 00000000000..24d9885a12c --- /dev/null +++ b/event_system_test.go @@ -0,0 +1,164 @@ +package main + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/Sirupsen/logrus" + + "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/config" +) + +var ( + handlerTypes = []apidef.TykEventHandlerName{ + EH_LogHandler, + EH_WebHook, + } +) + +func prepareSpecWithEvents(logger *logrus.Logger) (spec *APISpec) { + if logger == nil { + logger = log + } + def := &apidef.APIDefinition{ + EventHandlers: apidef.EventHandlerMetaConfig{ + Events: map[apidef.TykEvent][]apidef.EventHandlerTriggerConfig{ + EventAuthFailure: { + { + Handler: EH_LogHandler, + HandlerMeta: map[string]interface{}{ + "prefix": "testprefix", + "logger": logger, + }, + }, + }, + }, + }, + } + + spec = &APISpec{APIDefinition: def} + // From api_definition.go: + spec.EventPaths = make(map[apidef.TykEvent][]config.TykEventHandler) + for eventName, eventHandlerConfs := range def.EventHandlers.Events { + for _, handlerConf := range eventHandlerConfs { + eventHandlerInstance, err := EventHandlerByName(handlerConf, spec) + + if err != nil { + log.Error("Failed to init event handler: ", err) + } else { + spec.EventPaths[eventName] = append(spec.EventPaths[eventName], eventHandlerInstance) + } + + } + } + + return spec +} + +func prepareEventsConf() (conf *config.Config) { + return &config.Config{ + EventHandlers: apidef.EventHandlerMetaConfig{ + Events: map[apidef.TykEvent][]apidef.EventHandlerTriggerConfig{ + EventQuotaExceeded: { + { + Handler: EH_LogHandler, + HandlerMeta: map[string]interface{}{ + "prefix": "testprefix1", + "logger": log, + }, + }, + }, + EventAuthFailure: { + { + Handler: EH_LogHandler, + HandlerMeta: map[string]interface{}{ + "prefix": "testprefix2", + "logger": log, + }, + }, + }, + }, + }, + } +} + +func prepareEventHandlerConfig(handler apidef.TykEventHandlerName) (config apidef.EventHandlerTriggerConfig) { + config.Handler = handler + switch handler { + case EH_LogHandler: + config.HandlerMeta = map[string]interface{}{ + "prefix": "testprefix", + "logger": log, + } + case EH_WebHook: + config.HandlerMeta = map[string]interface{}{} + } + return config +} +func TestEventHandlerByName(t *testing.T) { + spec := prepareSpecWithEvents(nil) + for _, handlerType := range handlerTypes { + handlerConfig := prepareEventHandlerConfig(handlerType) + _, err := EventHandlerByName(handlerConfig, spec) + + // CP is disabled on standard builds: + if handlerType == EH_CoProcessHandler { + continue + } + if err != nil { + t.Fatalf("Couldn't get handler for %s\n", handlerType) + } + } +} + +func TestLogMessageEventHandler(t *testing.T) { + buf := &bytes.Buffer{} + testLogger := logrus.New() + testLogger.Out = buf + spec := prepareSpecWithEvents(testLogger) + handler := spec.EventPaths[EventAuthFailure][0] + em := config.EventMessage{ + Type: EventAuthFailure, + Meta: EventKeyFailureMeta{ + EventMetaDefault: EventMetaDefault{Message: "Auth Failure"}, + Path: "/", + Origin: "127.0.0.1", + Key: "abc", + }, + TimeStamp: time.Now().Local().String(), + } + lookup := "testprefix:AuthFailure" + handler.HandleEvent(em) + if !strings.Contains(buf.String(), lookup) { + t.Fatal("Couldn't find log message") + } +} + +func TestInitGenericEventHandlers(t *testing.T) { + conf := prepareEventsConf() + initGenericEventHandlers(conf) + triggers := conf.EventTriggers + if len(triggers) != 2 { + t.Fatal("EventTriggers length doesn't match") + } + for _, e := range []apidef.TykEvent{EventQuotaExceeded, EventAuthFailure} { + if triggers[e] == nil { + t.Fatalf("EventTriggers doesn't contain %s handlers", e) + } + } + for _, handlers := range triggers { + if len(handlers) != 1 { + t.Fatal("EventTriggers handlers length doesn't match") + } + } +} + +func BenchmarkInitGenericEventHandlers(b *testing.B) { + conf := prepareEventsConf() + for i := 0; i < b.N; i++ { + initGenericEventHandlers(conf) + } +} diff --git a/main.go b/main.go index cae95dde5c0..90f44976411 100644 --- a/main.go +++ b/main.go @@ -189,11 +189,6 @@ func setupGlobals() { templatesDir := filepath.Join(config.Global().TemplatePath, "error*") templates = template.Must(template.ParseGlob(templatesDir)) - // Set up global JSVM - if config.Global().EnableJSVM { - GlobalEventsJSVM.Init(nil) - } - if config.Global().CoProcessOptions.EnableCoProcess { if err := CoProcessInit(); err != nil { log.WithField("prefix", "coprocess").Error(err) @@ -609,6 +604,11 @@ func doReload() { reloadMu.Lock() defer reloadMu.Unlock() + // Initialize/reset the JSVM + if config.Global().EnableJSVM { + GlobalEventsJSVM.Init(nil) + } + // Load the API Policies syncPolicies() // load the specs @@ -621,12 +621,6 @@ func doReload() { } // We have updated specs, lets load those... - - // Reset the JSVM - if config.Global().EnableJSVM { - GlobalEventsJSVM.Init(nil) - } - mainLog.Info("Preparing new router") newRouter := mux.NewRouter() if config.Global().HttpServerOptions.OverrideDefaults { @@ -893,7 +887,7 @@ func afterConfSetup(conf *config.Config) { GlobalRPCPingTimeout = time.Second * time.Duration(conf.SlaveOptions.PingTimeout) GlobalRPCCallTimeout = time.Second * time.Duration(conf.SlaveOptions.CallTimeout) - conf.EventTriggers = InitGenericEventHandlers(conf.EventHandlers) + initGenericEventHandlers(conf) } var hostDetails struct { From 31bd9d7e0103eb1bb8cf3e3186e0be2d66ded3f4 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 19 May 2018 08:07:33 +0300 Subject: [PATCH 09/54] Lower logs verbosity - Running multiple Tyk group in same Redis is a normal setup (while not very common). In this case DRL will receive messages from different groups, and will ignore them. - RPC analytics purger can fail in case of MDCB auto-scaling events or session timeout. Should not be error log. Additionally fixed setting log level though tyk config. It was setting log level only for `mainLog` which used only for initialization. --- distributed_rate_limiter.go | 2 +- main.go | 6 +++--- rpc_analytics_purger.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/distributed_rate_limiter.go b/distributed_rate_limiter.go index 894c8b4f6ba..913529bcdb6 100644 --- a/distributed_rate_limiter.go +++ b/distributed_rate_limiter.go @@ -94,7 +94,7 @@ func onServerStatusReceivedHandler(payload string) { if err := DRLManager.AddOrUpdateServer(serverData); err != nil { log.WithError(err). WithField("serverData", serverData). - Error("AddOrUpdateServer error") + Debug("AddOrUpdateServer error. Seems like you running multiple segmented Tyk groups in same Redis.") return } log.Debug(DRLManager.Report()) diff --git a/main.go b/main.go index 90f44976411..ed3206d8350 100644 --- a/main.go +++ b/main.go @@ -827,11 +827,11 @@ func initialiseSystem() error { case "", "info": // default, do nothing case "error": - mainLog.Level = logrus.ErrorLevel + log.Level = logrus.ErrorLevel case "warn": - mainLog.Level = logrus.WarnLevel + log.Level = logrus.WarnLevel case "debug": - mainLog.Level = logrus.DebugLevel + log.Level = logrus.DebugLevel default: mainLog.Fatalf("Invalid log level %q specified in config, must be error, warn, debug or info. ", level) } diff --git a/rpc_analytics_purger.go b/rpc_analytics_purger.go index 7599df7643c..658edef0431 100644 --- a/rpc_analytics_purger.go +++ b/rpc_analytics_purger.go @@ -72,7 +72,7 @@ func (r *RPCPurger) PurgeCache() { // Send keys to RPC if _, err := RPCFuncClientSingleton.Call("PurgeAnalyticsData", string(data)); err != nil { emitRPCErrorEvent(rpcFuncClientSingletonCall, "PurgeAnalyticsData", err) - log.Error("Failed to call purge: ", err) + log.Warn("Failed to call purge, retrying: ", err) } } From 2d7cba839656ef9bcceecf68ba0dc8e89528200f Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 19 May 2018 09:49:16 +0300 Subject: [PATCH 10/54] Do not require client certificate unless necessary Client cert now asked only if one APIs, which share same domain, require this feature. Fix https://github.com/TykTechnologies/tyk/issues/1381 --- cert.go | 11 ++++++++++ cert_test.go | 62 ---------------------------------------------------- main.go | 2 +- 3 files changed, 12 insertions(+), 63 deletions(-) diff --git a/cert.go b/cert.go index d4313a43ed7..1cdec52bd3d 100644 --- a/cert.go +++ b/cert.go @@ -226,6 +226,17 @@ func getTLSConfigForClient(baseConfig *tls.Config, listenPort int) func(hello *t break } } + + // No mutual tls APIs with matched domain found + // Check if one of APIs without domain, require asking client cert + if newConfig.ClientAuth == tls.NoClientCert { + for _, spec := range apiSpecs { + if spec.Auth.UseCertificate || (spec.Domain == "" && spec.UseMutualTLSAuth) { + newConfig.ClientAuth = tls.RequestClientCert + break + } + } + } apisMu.RUnlock() return newConfig, nil diff --git a/cert_test.go b/cert_test.go index d820a3163e1..9c333c4f132 100644 --- a/cert_test.go +++ b/cert_test.go @@ -253,68 +253,6 @@ func TestGatewayControlAPIMutualTLS(t *testing.T) { Path: "/tyk/certs", Code: 200, ControlRequest: true, AdminAuth: true, Client: clientWithCert, }) }) - - t.Run("Same domain", func(t *testing.T) { - certID, _ := CertificateManager.Add(combinedPEM, "") - defer CertificateManager.Delete(certID) - - globalConf = config.Global() - globalConf.ControlAPIHostname = "localhost" - globalConf.HttpServerOptions.SSLCertificates = []string{certID} - config.SetGlobal(globalConf) - - defer func() { - globalConf = config.Global() - globalConf.HttpServerOptions.SSLCertificates = nil - globalConf.Security.Certificates.ControlAPI = nil - config.SetGlobal(globalConf) - CertificateManager.FlushCache() - - }() - - ts := newTykTestServer() - defer ts.Close() - - certNotMatchErr := `Certificate with SHA256 ` + certs.HexSHA256(clientCert.Certificate[0]) + ` not allowed` - - t.Run("Without or not valid certificates", func(t *testing.T) { - ts.Run(t, []test.TestCase{ - // Should acess tyk without client certificates - {Client: clientWithoutCert}, - - // Error for client without certificate - {Path: "/tyk/certs", AdminAuth: true, Code: 403, BodyMatch: `"message":"Client TLS certificate is required"`, Client: clientWithoutCert}, - - // Error for client with unknown certificate - {Path: "/tyk/certs", AdminAuth: true, Code: 403, BodyMatch: `"message":"` + certNotMatchErr, Client: clientWithCert}, - }...) - }) - - t.Run("Redis certificate", func(t *testing.T) { - clientCertID, _ := CertificateManager.Add(clientCertPem, "") - defer CertificateManager.Delete(clientCertID) - globalConf = config.Global() - globalConf.Security.Certificates.ControlAPI = []string{clientCertID} - config.SetGlobal(globalConf) - - ts.Run(t, []test.TestCase{ - {Path: "/tyk/certs", AdminAuth: true, Code: 200, Client: clientWithCert}, - }...) - }) - - t.Run("File certificate", func(t *testing.T) { - certPath := filepath.Join(dir, "client.pem") - ioutil.WriteFile(certPath, clientCertPem, 0666) - - globalConf = config.Global() - globalConf.Security.Certificates.ControlAPI = []string{certPath} - config.SetGlobal(globalConf) - - ts.Run(t, []test.TestCase{ - {Path: "/tyk/certs", AdminAuth: true, Code: 200, Client: clientWithCert}, - }...) - }) - }) } func TestAPIMutualTLS(t *testing.T) { diff --git a/main.go b/main.go index ed3206d8350..c2050f396b2 100644 --- a/main.go +++ b/main.go @@ -1132,7 +1132,7 @@ func generateListener(listenPort int) (net.Listener, error) { GetCertificate: dummyGetCertificate, ServerName: httpServerOptions.ServerName, MinVersion: httpServerOptions.MinVersion, - ClientAuth: tls.RequestClientCert, + ClientAuth: tls.NoClientCert, InsecureSkipVerify: httpServerOptions.SSLInsecureSkipVerify, CipherSuites: getCipherAliases(httpServerOptions.Ciphers), } From f41cd8f57d79626ba278c0bbc86882ce59ca5e50 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Fri, 4 May 2018 17:53:13 -0400 Subject: [PATCH 11/54] vendor: update gRPC, Protocol Buffers and other dependencies Updates the following dependencies as well (required by the gRPC and Protocol Buffers package) - golang.org/x/net/context - golang.org/x/net/http2 - golang.org/x/net/trace --- .../github.com/golang/protobuf/proto/Makefile | 43 - .../github.com/golang/protobuf/proto/clone.go | 46 +- .../golang/protobuf/proto/decode.go | 668 +--- .../golang/protobuf/proto/discard.go | 350 +++ .../golang/protobuf/proto/encode.go | 1189 +------- .../github.com/golang/protobuf/proto/equal.go | 30 +- .../golang/protobuf/proto/extensions.go | 204 +- .../github.com/golang/protobuf/proto/lib.go | 71 +- .../golang/protobuf/proto/message_set.go | 81 +- .../golang/protobuf/proto/pointer_reflect.go | 595 ++-- .../golang/protobuf/proto/pointer_unsafe.go | 366 ++- .../golang/protobuf/proto/properties.go | 424 +-- .../golang/protobuf/proto/table_marshal.go | 2681 +++++++++++++++++ .../golang/protobuf/proto/table_merge.go | 654 ++++ .../golang/protobuf/proto/table_unmarshal.go | 1967 ++++++++++++ .../github.com/golang/protobuf/proto/text.go | 61 +- .../golang/protobuf/proto/text_parser.go | 75 +- vendor/golang.org/x/net/context/context.go | 104 +- vendor/golang.org/x/net/context/go19.go | 20 + vendor/golang.org/x/net/context/pre_go19.go | 109 + vendor/golang.org/x/net/http2/ciphers.go | 641 ++++ .../x/net/http2/configure_transport.go | 4 +- vendor/golang.org/x/net/http2/databuffer.go | 146 + vendor/golang.org/x/net/http2/errors.go | 13 +- vendor/golang.org/x/net/http2/fixed_buffer.go | 60 - vendor/golang.org/x/net/http2/frame.go | 55 +- vendor/golang.org/x/net/http2/go16.go | 27 - vendor/golang.org/x/net/http2/go18.go | 8 +- vendor/golang.org/x/net/http2/go19.go | 16 + vendor/golang.org/x/net/http2/hpack/encode.go | 29 +- vendor/golang.org/x/net/http2/hpack/hpack.go | 100 +- vendor/golang.org/x/net/http2/hpack/tables.go | 255 +- vendor/golang.org/x/net/http2/http2.go | 10 +- vendor/golang.org/x/net/http2/not_go16.go | 25 - vendor/golang.org/x/net/http2/not_go18.go | 2 + vendor/golang.org/x/net/http2/not_go19.go | 16 + vendor/golang.org/x/net/http2/pipe.go | 16 +- vendor/golang.org/x/net/http2/server.go | 575 ++-- vendor/golang.org/x/net/http2/transport.go | 547 ++-- vendor/golang.org/x/net/http2/write.go | 7 +- .../x/net/http2/writesched_priority.go | 2 +- vendor/golang.org/x/net/trace/events.go | 6 +- vendor/golang.org/x/net/trace/trace.go | 130 +- vendor/golang.org/x/net/trace/trace_go16.go | 21 + vendor/golang.org/x/net/trace/trace_go17.go | 21 + vendor/google.golang.org/grpc/AUTHORS | 1 + vendor/google.golang.org/grpc/CONTRIBUTING.md | 68 +- vendor/google.golang.org/grpc/LICENSE | 230 +- vendor/google.golang.org/grpc/Makefile | 22 +- vendor/google.golang.org/grpc/PATENTS | 22 - vendor/google.golang.org/grpc/README.md | 32 +- vendor/google.golang.org/grpc/backoff.go | 32 +- vendor/google.golang.org/grpc/balancer.go | 82 +- .../grpc/balancer/balancer.go | 228 ++ .../grpc/balancer_conn_wrappers.go | 300 ++ .../grpc/balancer_v1_wrapper.go | 372 +++ vendor/google.golang.org/grpc/call.go | 301 +- .../google.golang.org/grpc/channelz/funcs.go | 573 ++++ .../google.golang.org/grpc/channelz/types.go | 418 +++ vendor/google.golang.org/grpc/clientconn.go | 1589 +++++++--- vendor/google.golang.org/grpc/codec.go | 50 + vendor/google.golang.org/grpc/codegen.sh | 2 +- .../grpc/codes/code_string.go | 66 +- vendor/google.golang.org/grpc/codes/codes.go | 103 +- vendor/google.golang.org/grpc/coverage.sh | 47 - .../grpc/credentials/credentials.go | 82 +- .../grpc/credentials/credentials_util_go17.go | 38 +- .../grpc/credentials/credentials_util_go18.go | 38 + .../credentials/credentials_util_pre_go17.go | 37 +- vendor/google.golang.org/grpc/doc.go | 20 +- .../grpc/encoding/encoding.go | 118 + .../grpc/encoding/proto/proto.go | 110 + vendor/google.golang.org/grpc/envconfig.go | 37 + vendor/google.golang.org/grpc/go16.go | 70 + vendor/google.golang.org/grpc/go17.go | 71 + vendor/google.golang.org/grpc/grpclb.go | 341 +++ .../google.golang.org/grpc/grpclb_picker.go | 159 + .../grpc/grpclb_remote_balancer.go | 266 ++ vendor/google.golang.org/grpc/grpclb_util.go | 214 ++ .../google.golang.org/grpc/grpclog/grpclog.go | 126 + .../google.golang.org/grpc/grpclog/logger.go | 108 +- .../grpc/grpclog/loggerv2.go | 195 ++ vendor/google.golang.org/grpc/interceptor.go | 47 +- .../grpc/metadata/metadata.go | 213 +- vendor/google.golang.org/grpc/peer/peer.go | 38 +- .../google.golang.org/grpc/picker_wrapper.go | 331 ++ vendor/google.golang.org/grpc/pickfirst.go | 108 + vendor/google.golang.org/grpc/proxy.go | 130 + .../grpc/resolver_conn_wrapper.go | 158 + vendor/google.golang.org/grpc/rpc_util.go | 692 +++-- vendor/google.golang.org/grpc/server.go | 1087 +++++-- .../google.golang.org/grpc/service_config.go | 233 ++ .../google.golang.org/grpc/stats/handlers.go | 154 +- vendor/google.golang.org/grpc/stats/stats.go | 149 +- .../google.golang.org/grpc/status/status.go | 189 ++ vendor/google.golang.org/grpc/stream.go | 759 +++-- vendor/google.golang.org/grpc/trace.go | 50 +- .../grpc/transport/bdp_estimator.go | 140 + .../grpc/transport/control.go | 193 -- .../grpc/transport/controlbuf.go | 769 +++++ .../grpc/transport/flowcontrol.go | 236 ++ .../google.golang.org/grpc/transport/go16.go | 55 +- .../google.golang.org/grpc/transport/go17.go | 58 +- .../grpc/transport/handler_server.go | 196 +- .../grpc/transport/http2_client.go | 1548 +++++----- .../grpc/transport/http2_server.go | 1222 +++++--- .../grpc/transport/http_util.go | 487 +-- .../google.golang.org/grpc/transport/log.go | 50 + .../grpc/transport/pre_go16.go | 51 - .../grpc/transport/transport.go | 503 ++-- vendor/google.golang.org/grpc/vet.sh | 93 + vendor/vendor.json | 108 +- 112 files changed, 20925 insertions(+), 8160 deletions(-) delete mode 100644 vendor/github.com/golang/protobuf/proto/Makefile create mode 100644 vendor/github.com/golang/protobuf/proto/discard.go create mode 100644 vendor/github.com/golang/protobuf/proto/table_marshal.go create mode 100644 vendor/github.com/golang/protobuf/proto/table_merge.go create mode 100644 vendor/github.com/golang/protobuf/proto/table_unmarshal.go create mode 100644 vendor/golang.org/x/net/context/go19.go create mode 100644 vendor/golang.org/x/net/context/pre_go19.go create mode 100644 vendor/golang.org/x/net/http2/ciphers.go create mode 100644 vendor/golang.org/x/net/http2/databuffer.go delete mode 100644 vendor/golang.org/x/net/http2/fixed_buffer.go create mode 100644 vendor/golang.org/x/net/http2/go19.go create mode 100644 vendor/golang.org/x/net/http2/not_go19.go create mode 100644 vendor/golang.org/x/net/trace/trace_go16.go create mode 100644 vendor/golang.org/x/net/trace/trace_go17.go create mode 100644 vendor/google.golang.org/grpc/AUTHORS delete mode 100644 vendor/google.golang.org/grpc/PATENTS create mode 100644 vendor/google.golang.org/grpc/balancer/balancer.go create mode 100644 vendor/google.golang.org/grpc/balancer_conn_wrappers.go create mode 100644 vendor/google.golang.org/grpc/balancer_v1_wrapper.go create mode 100644 vendor/google.golang.org/grpc/channelz/funcs.go create mode 100644 vendor/google.golang.org/grpc/channelz/types.go create mode 100644 vendor/google.golang.org/grpc/codec.go delete mode 100755 vendor/google.golang.org/grpc/coverage.sh create mode 100644 vendor/google.golang.org/grpc/credentials/credentials_util_go18.go create mode 100644 vendor/google.golang.org/grpc/encoding/encoding.go create mode 100644 vendor/google.golang.org/grpc/encoding/proto/proto.go create mode 100644 vendor/google.golang.org/grpc/envconfig.go create mode 100644 vendor/google.golang.org/grpc/go16.go create mode 100644 vendor/google.golang.org/grpc/go17.go create mode 100644 vendor/google.golang.org/grpc/grpclb.go create mode 100644 vendor/google.golang.org/grpc/grpclb_picker.go create mode 100644 vendor/google.golang.org/grpc/grpclb_remote_balancer.go create mode 100644 vendor/google.golang.org/grpc/grpclb_util.go create mode 100644 vendor/google.golang.org/grpc/grpclog/grpclog.go create mode 100644 vendor/google.golang.org/grpc/grpclog/loggerv2.go create mode 100644 vendor/google.golang.org/grpc/picker_wrapper.go create mode 100644 vendor/google.golang.org/grpc/pickfirst.go create mode 100644 vendor/google.golang.org/grpc/proxy.go create mode 100644 vendor/google.golang.org/grpc/resolver_conn_wrapper.go create mode 100644 vendor/google.golang.org/grpc/service_config.go create mode 100644 vendor/google.golang.org/grpc/status/status.go create mode 100644 vendor/google.golang.org/grpc/transport/bdp_estimator.go delete mode 100644 vendor/google.golang.org/grpc/transport/control.go create mode 100644 vendor/google.golang.org/grpc/transport/controlbuf.go create mode 100644 vendor/google.golang.org/grpc/transport/flowcontrol.go create mode 100644 vendor/google.golang.org/grpc/transport/log.go delete mode 100644 vendor/google.golang.org/grpc/transport/pre_go16.go create mode 100755 vendor/google.golang.org/grpc/vet.sh diff --git a/vendor/github.com/golang/protobuf/proto/Makefile b/vendor/github.com/golang/protobuf/proto/Makefile deleted file mode 100644 index e2e0651a934..00000000000 --- a/vendor/github.com/golang/protobuf/proto/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -# Go support for Protocol Buffers - Google's data interchange format -# -# Copyright 2010 The Go Authors. All rights reserved. -# https://github.com/golang/protobuf -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -install: - go install - -test: install generate-test-pbs - go test - - -generate-test-pbs: - make install - make -C testdata - protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata,Mgoogle/protobuf/any.proto=github.com/golang/protobuf/ptypes/any:. proto3_proto/proto3.proto - make diff --git a/vendor/github.com/golang/protobuf/proto/clone.go b/vendor/github.com/golang/protobuf/proto/clone.go index e392575b353..3cd3249f706 100644 --- a/vendor/github.com/golang/protobuf/proto/clone.go +++ b/vendor/github.com/golang/protobuf/proto/clone.go @@ -35,22 +35,39 @@ package proto import ( + "fmt" "log" "reflect" "strings" ) // Clone returns a deep copy of a protocol buffer. -func Clone(pb Message) Message { - in := reflect.ValueOf(pb) +func Clone(src Message) Message { + in := reflect.ValueOf(src) if in.IsNil() { - return pb + return src } - out := reflect.New(in.Type().Elem()) - // out is empty so a merge is a deep copy. - mergeStruct(out.Elem(), in.Elem()) - return out.Interface().(Message) + dst := out.Interface().(Message) + Merge(dst, src) + return dst +} + +// Merger is the interface representing objects that can merge messages of the same type. +type Merger interface { + // Merge merges src into this message. + // Required and optional fields that are set in src will be set to that value in dst. + // Elements of repeated fields will be appended. + // + // Merge may panic if called with a different argument type than the receiver. + Merge(src Message) +} + +// generatedMerger is the custom merge method that generated protos will have. +// We must add this method since a generate Merge method will conflict with +// many existing protos that have a Merge data field already defined. +type generatedMerger interface { + XXX_Merge(src Message) } // Merge merges src into dst. @@ -58,17 +75,24 @@ func Clone(pb Message) Message { // Elements of repeated fields will be appended. // Merge panics if src and dst are not the same type, or if dst is nil. func Merge(dst, src Message) { + if m, ok := dst.(Merger); ok { + m.Merge(src) + return + } + in := reflect.ValueOf(src) out := reflect.ValueOf(dst) if out.IsNil() { panic("proto: nil destination") } if in.Type() != out.Type() { - // Explicit test prior to mergeStruct so that mistyped nils will fail - panic("proto: type mismatch") + panic(fmt.Sprintf("proto.Merge(%T, %T) type mismatch", dst, src)) } if in.IsNil() { - // Merging nil into non-nil is a quiet no-op + return // Merge from nil src is a noop + } + if m, ok := dst.(generatedMerger); ok { + m.XXX_Merge(src) return } mergeStruct(out.Elem(), in.Elem()) @@ -84,7 +108,7 @@ func mergeStruct(out, in reflect.Value) { mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i]) } - if emIn, ok := extendable(in.Addr().Interface()); ok { + if emIn, err := extendable(in.Addr().Interface()); err == nil { emOut, _ := extendable(out.Addr().Interface()) mIn, muIn := emIn.extensionsRead() if mIn != nil { diff --git a/vendor/github.com/golang/protobuf/proto/decode.go b/vendor/github.com/golang/protobuf/proto/decode.go index aa207298f99..d9aa3c42d66 100644 --- a/vendor/github.com/golang/protobuf/proto/decode.go +++ b/vendor/github.com/golang/protobuf/proto/decode.go @@ -39,8 +39,6 @@ import ( "errors" "fmt" "io" - "os" - "reflect" ) // errOverflow is returned when an integer is too large to be represented. @@ -50,10 +48,6 @@ var errOverflow = errors.New("proto: integer overflow") // wire type is encountered. It does not get returned to user code. var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for oneof") -// The fundamental decoders that interpret bytes on the wire. -// Those that take integer types all return uint64 and are -// therefore of type valueDecoder. - // DecodeVarint reads a varint-encoded integer from the slice. // It returns the integer and the number of bytes consumed, or // zero if there is not enough. @@ -267,9 +261,6 @@ func (p *Buffer) DecodeZigzag32() (x uint64, err error) { return } -// These are not ValueDecoders: they produce an array of bytes or a string. -// bytes, embedded messages - // DecodeRawBytes reads a count-delimited byte buffer from the Buffer. // This is the format used for the bytes protocol buffer // type and for embedded messages. @@ -311,81 +302,29 @@ func (p *Buffer) DecodeStringBytes() (s string, err error) { return string(buf), nil } -// Skip the next item in the buffer. Its wire type is decoded and presented as an argument. -// If the protocol buffer has extensions, and the field matches, add it as an extension. -// Otherwise, if the XXX_unrecognized field exists, append the skipped data there. -func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base structPointer, unrecField field) error { - oi := o.index - - err := o.skip(t, tag, wire) - if err != nil { - return err - } - - if !unrecField.IsValid() { - return nil - } - - ptr := structPointer_Bytes(base, unrecField) - - // Add the skipped field to struct field - obuf := o.buf - - o.buf = *ptr - o.EncodeVarint(uint64(tag<<3 | wire)) - *ptr = append(o.buf, obuf[oi:o.index]...) - - o.buf = obuf - - return nil -} - -// Skip the next item in the buffer. Its wire type is decoded and presented as an argument. -func (o *Buffer) skip(t reflect.Type, tag, wire int) error { - - var u uint64 - var err error - - switch wire { - case WireVarint: - _, err = o.DecodeVarint() - case WireFixed64: - _, err = o.DecodeFixed64() - case WireBytes: - _, err = o.DecodeRawBytes(false) - case WireFixed32: - _, err = o.DecodeFixed32() - case WireStartGroup: - for { - u, err = o.DecodeVarint() - if err != nil { - break - } - fwire := int(u & 0x7) - if fwire == WireEndGroup { - break - } - ftag := int(u >> 3) - err = o.skip(t, ftag, fwire) - if err != nil { - break - } - } - default: - err = fmt.Errorf("proto: can't skip unknown wire type %d for %s", wire, t) - } - return err -} - // Unmarshaler is the interface representing objects that can -// unmarshal themselves. The method should reset the receiver before -// decoding starts. The argument points to data that may be +// unmarshal themselves. The argument points to data that may be // overwritten, so implementations should not keep references to the // buffer. +// Unmarshal implementations should not clear the receiver. +// Any unmarshaled data should be merged into the receiver. +// Callers of Unmarshal that do not want to retain existing data +// should Reset the receiver before calling Unmarshal. type Unmarshaler interface { Unmarshal([]byte) error } +// newUnmarshaler is the interface representing objects that can +// unmarshal themselves. The semantics are identical to Unmarshaler. +// +// This exists to support protoc-gen-go generated messages. +// The proto package will stop type-asserting to this interface in the future. +// +// DO NOT DEPEND ON THIS. +type newUnmarshaler interface { + XXX_Unmarshal([]byte) error +} + // Unmarshal parses the protocol buffer representation in buf and places the // decoded result in pb. If the struct underlying pb does not match // the data in buf, the results can be unpredictable. @@ -395,7 +334,13 @@ type Unmarshaler interface { // to preserve and append to existing data. func Unmarshal(buf []byte, pb Message) error { pb.Reset() - return UnmarshalMerge(buf, pb) + if u, ok := pb.(newUnmarshaler); ok { + return u.XXX_Unmarshal(buf) + } + if u, ok := pb.(Unmarshaler); ok { + return u.Unmarshal(buf) + } + return NewBuffer(buf).Unmarshal(pb) } // UnmarshalMerge parses the protocol buffer representation in buf and @@ -405,8 +350,16 @@ func Unmarshal(buf []byte, pb Message) error { // UnmarshalMerge merges into existing data in pb. // Most code should use Unmarshal instead. func UnmarshalMerge(buf []byte, pb Message) error { - // If the object can unmarshal itself, let it. + if u, ok := pb.(newUnmarshaler); ok { + return u.XXX_Unmarshal(buf) + } if u, ok := pb.(Unmarshaler); ok { + // NOTE: The history of proto have unfortunately been inconsistent + // whether Unmarshaler should or should not implicitly clear itself. + // Some implementations do, most do not. + // Thus, calling this here may or may not do what people want. + // + // See https://github.com/golang/protobuf/issues/424 return u.Unmarshal(buf) } return NewBuffer(buf).Unmarshal(pb) @@ -422,12 +375,17 @@ func (p *Buffer) DecodeMessage(pb Message) error { } // DecodeGroup reads a tag-delimited group from the Buffer. +// StartGroup tag is already consumed. This function consumes +// EndGroup tag. func (p *Buffer) DecodeGroup(pb Message) error { - typ, base, err := getbase(pb) - if err != nil { - return err + b := p.buf[p.index:] + x, y := findEndGroup(b) + if x < 0 { + return io.ErrUnexpectedEOF } - return p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), true, base) + err := Unmarshal(b[:x], pb) + p.index += y + return err } // Unmarshal parses the protocol buffer representation in the @@ -438,533 +396,33 @@ func (p *Buffer) DecodeGroup(pb Message) error { // Unlike proto.Unmarshal, this does not reset pb before starting to unmarshal. func (p *Buffer) Unmarshal(pb Message) error { // If the object can unmarshal itself, let it. - if u, ok := pb.(Unmarshaler); ok { - err := u.Unmarshal(p.buf[p.index:]) + if u, ok := pb.(newUnmarshaler); ok { + err := u.XXX_Unmarshal(p.buf[p.index:]) p.index = len(p.buf) return err } - - typ, base, err := getbase(pb) - if err != nil { - return err - } - - err = p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), false, base) - - if collectStats { - stats.Decode++ - } - - return err -} - -// unmarshalType does the work of unmarshaling a structure. -func (o *Buffer) unmarshalType(st reflect.Type, prop *StructProperties, is_group bool, base structPointer) error { - var state errorState - required, reqFields := prop.reqCount, uint64(0) - - var err error - for err == nil && o.index < len(o.buf) { - oi := o.index - var u uint64 - u, err = o.DecodeVarint() - if err != nil { - break - } - wire := int(u & 0x7) - if wire == WireEndGroup { - if is_group { - if required > 0 { - // Not enough information to determine the exact field. - // (See below.) - return &RequiredNotSetError{"{Unknown}"} - } - return nil // input is satisfied - } - return fmt.Errorf("proto: %s: wiretype end group for non-group", st) - } - tag := int(u >> 3) - if tag <= 0 { - return fmt.Errorf("proto: %s: illegal tag %d (wire type %d)", st, tag, wire) - } - fieldnum, ok := prop.decoderTags.get(tag) - if !ok { - // Maybe it's an extension? - if prop.extendable { - if e, _ := extendable(structPointer_Interface(base, st)); isExtensionField(e, int32(tag)) { - if err = o.skip(st, tag, wire); err == nil { - extmap := e.extensionsWrite() - ext := extmap[int32(tag)] // may be missing - ext.enc = append(ext.enc, o.buf[oi:o.index]...) - extmap[int32(tag)] = ext - } - continue - } - } - // Maybe it's a oneof? - if prop.oneofUnmarshaler != nil { - m := structPointer_Interface(base, st).(Message) - // First return value indicates whether tag is a oneof field. - ok, err = prop.oneofUnmarshaler(m, tag, wire, o) - if err == ErrInternalBadWireType { - // Map the error to something more descriptive. - // Do the formatting here to save generated code space. - err = fmt.Errorf("bad wiretype for oneof field in %T", m) - } - if ok { - continue - } - } - err = o.skipAndSave(st, tag, wire, base, prop.unrecField) - continue - } - p := prop.Prop[fieldnum] - - if p.dec == nil { - fmt.Fprintf(os.Stderr, "proto: no protobuf decoder for %s.%s\n", st, st.Field(fieldnum).Name) - continue - } - dec := p.dec - if wire != WireStartGroup && wire != p.WireType { - if wire == WireBytes && p.packedDec != nil { - // a packable field - dec = p.packedDec - } else { - err = fmt.Errorf("proto: bad wiretype for field %s.%s: got wiretype %d, want %d", st, st.Field(fieldnum).Name, wire, p.WireType) - continue - } - } - decErr := dec(o, p, base) - if decErr != nil && !state.shouldContinue(decErr, p) { - err = decErr - } - if err == nil && p.Required { - // Successfully decoded a required field. - if tag <= 64 { - // use bitmap for fields 1-64 to catch field reuse. - var mask uint64 = 1 << uint64(tag-1) - if reqFields&mask == 0 { - // new required field - reqFields |= mask - required-- - } - } else { - // This is imprecise. It can be fooled by a required field - // with a tag > 64 that is encoded twice; that's very rare. - // A fully correct implementation would require allocating - // a data structure, which we would like to avoid. - required-- - } - } - } - if err == nil { - if is_group { - return io.ErrUnexpectedEOF - } - if state.err != nil { - return state.err - } - if required > 0 { - // Not enough information to determine the exact field. If we use extra - // CPU, we could determine the field only if the missing required field - // has a tag <= 64 and we check reqFields. - return &RequiredNotSetError{"{Unknown}"} - } - } - return err -} - -// Individual type decoders -// For each, -// u is the decoded value, -// v is a pointer to the field (pointer) in the struct - -// Sizes of the pools to allocate inside the Buffer. -// The goal is modest amortization and allocation -// on at least 16-byte boundaries. -const ( - boolPoolSize = 16 - uint32PoolSize = 8 - uint64PoolSize = 4 -) - -// Decode a bool. -func (o *Buffer) dec_bool(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - if len(o.bools) == 0 { - o.bools = make([]bool, boolPoolSize) - } - o.bools[0] = u != 0 - *structPointer_Bool(base, p.field) = &o.bools[0] - o.bools = o.bools[1:] - return nil -} - -func (o *Buffer) dec_proto3_bool(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - *structPointer_BoolVal(base, p.field) = u != 0 - return nil -} - -// Decode an int32. -func (o *Buffer) dec_int32(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - word32_Set(structPointer_Word32(base, p.field), o, uint32(u)) - return nil -} - -func (o *Buffer) dec_proto3_int32(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - word32Val_Set(structPointer_Word32Val(base, p.field), uint32(u)) - return nil -} - -// Decode an int64. -func (o *Buffer) dec_int64(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - word64_Set(structPointer_Word64(base, p.field), o, u) - return nil -} - -func (o *Buffer) dec_proto3_int64(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - word64Val_Set(structPointer_Word64Val(base, p.field), o, u) - return nil -} - -// Decode a string. -func (o *Buffer) dec_string(p *Properties, base structPointer) error { - s, err := o.DecodeStringBytes() - if err != nil { - return err - } - *structPointer_String(base, p.field) = &s - return nil -} - -func (o *Buffer) dec_proto3_string(p *Properties, base structPointer) error { - s, err := o.DecodeStringBytes() - if err != nil { - return err - } - *structPointer_StringVal(base, p.field) = s - return nil -} - -// Decode a slice of bytes ([]byte). -func (o *Buffer) dec_slice_byte(p *Properties, base structPointer) error { - b, err := o.DecodeRawBytes(true) - if err != nil { - return err - } - *structPointer_Bytes(base, p.field) = b - return nil -} - -// Decode a slice of bools ([]bool). -func (o *Buffer) dec_slice_bool(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - v := structPointer_BoolSlice(base, p.field) - *v = append(*v, u != 0) - return nil -} - -// Decode a slice of bools ([]bool) in packed format. -func (o *Buffer) dec_slice_packed_bool(p *Properties, base structPointer) error { - v := structPointer_BoolSlice(base, p.field) - - nn, err := o.DecodeVarint() - if err != nil { - return err - } - nb := int(nn) // number of bytes of encoded bools - fin := o.index + nb - if fin < o.index { - return errOverflow - } - - y := *v - for o.index < fin { - u, err := p.valDec(o) - if err != nil { - return err - } - y = append(y, u != 0) - } - - *v = y - return nil -} - -// Decode a slice of int32s ([]int32). -func (o *Buffer) dec_slice_int32(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - structPointer_Word32Slice(base, p.field).Append(uint32(u)) - return nil -} - -// Decode a slice of int32s ([]int32) in packed format. -func (o *Buffer) dec_slice_packed_int32(p *Properties, base structPointer) error { - v := structPointer_Word32Slice(base, p.field) - - nn, err := o.DecodeVarint() - if err != nil { - return err - } - nb := int(nn) // number of bytes of encoded int32s - - fin := o.index + nb - if fin < o.index { - return errOverflow - } - for o.index < fin { - u, err := p.valDec(o) - if err != nil { - return err - } - v.Append(uint32(u)) - } - return nil -} - -// Decode a slice of int64s ([]int64). -func (o *Buffer) dec_slice_int64(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - - structPointer_Word64Slice(base, p.field).Append(u) - return nil -} - -// Decode a slice of int64s ([]int64) in packed format. -func (o *Buffer) dec_slice_packed_int64(p *Properties, base structPointer) error { - v := structPointer_Word64Slice(base, p.field) - - nn, err := o.DecodeVarint() - if err != nil { - return err - } - nb := int(nn) // number of bytes of encoded int64s - - fin := o.index + nb - if fin < o.index { - return errOverflow - } - for o.index < fin { - u, err := p.valDec(o) - if err != nil { - return err - } - v.Append(u) - } - return nil -} - -// Decode a slice of strings ([]string). -func (o *Buffer) dec_slice_string(p *Properties, base structPointer) error { - s, err := o.DecodeStringBytes() - if err != nil { - return err - } - v := structPointer_StringSlice(base, p.field) - *v = append(*v, s) - return nil -} - -// Decode a slice of slice of bytes ([][]byte). -func (o *Buffer) dec_slice_slice_byte(p *Properties, base structPointer) error { - b, err := o.DecodeRawBytes(true) - if err != nil { - return err - } - v := structPointer_BytesSlice(base, p.field) - *v = append(*v, b) - return nil -} - -// Decode a map field. -func (o *Buffer) dec_new_map(p *Properties, base structPointer) error { - raw, err := o.DecodeRawBytes(false) - if err != nil { - return err - } - oi := o.index // index at the end of this map entry - o.index -= len(raw) // move buffer back to start of map entry - - mptr := structPointer_NewAt(base, p.field, p.mtype) // *map[K]V - if mptr.Elem().IsNil() { - mptr.Elem().Set(reflect.MakeMap(mptr.Type().Elem())) - } - v := mptr.Elem() // map[K]V - - // Prepare addressable doubly-indirect placeholders for the key and value types. - // See enc_new_map for why. - keyptr := reflect.New(reflect.PtrTo(p.mtype.Key())).Elem() // addressable *K - keybase := toStructPointer(keyptr.Addr()) // **K - - var valbase structPointer - var valptr reflect.Value - switch p.mtype.Elem().Kind() { - case reflect.Slice: - // []byte - var dummy []byte - valptr = reflect.ValueOf(&dummy) // *[]byte - valbase = toStructPointer(valptr) // *[]byte - case reflect.Ptr: - // message; valptr is **Msg; need to allocate the intermediate pointer - valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V - valptr.Set(reflect.New(valptr.Type().Elem())) - valbase = toStructPointer(valptr) - default: - // everything else - valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V - valbase = toStructPointer(valptr.Addr()) // **V - } - - // Decode. - // This parses a restricted wire format, namely the encoding of a message - // with two fields. See enc_new_map for the format. - for o.index < oi { - // tagcode for key and value properties are always a single byte - // because they have tags 1 and 2. - tagcode := o.buf[o.index] - o.index++ - switch tagcode { - case p.mkeyprop.tagcode[0]: - if err := p.mkeyprop.dec(o, p.mkeyprop, keybase); err != nil { - return err - } - case p.mvalprop.tagcode[0]: - if err := p.mvalprop.dec(o, p.mvalprop, valbase); err != nil { - return err - } - default: - // TODO: Should we silently skip this instead? - return fmt.Errorf("proto: bad map data tag %d", raw[0]) - } - } - keyelem, valelem := keyptr.Elem(), valptr.Elem() - if !keyelem.IsValid() { - keyelem = reflect.Zero(p.mtype.Key()) - } - if !valelem.IsValid() { - valelem = reflect.Zero(p.mtype.Elem()) - } - - v.SetMapIndex(keyelem, valelem) - return nil -} - -// Decode a group. -func (o *Buffer) dec_struct_group(p *Properties, base structPointer) error { - bas := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(bas) { - // allocate new nested message - bas = toStructPointer(reflect.New(p.stype)) - structPointer_SetStructPointer(base, p.field, bas) - } - return o.unmarshalType(p.stype, p.sprop, true, bas) -} - -// Decode an embedded message. -func (o *Buffer) dec_struct_message(p *Properties, base structPointer) (err error) { - raw, e := o.DecodeRawBytes(false) - if e != nil { - return e - } - - bas := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(bas) { - // allocate new nested message - bas = toStructPointer(reflect.New(p.stype)) - structPointer_SetStructPointer(base, p.field, bas) - } - - // If the object can unmarshal itself, let it. - if p.isUnmarshaler { - iv := structPointer_Interface(bas, p.stype) - return iv.(Unmarshaler).Unmarshal(raw) - } - - obuf := o.buf - oi := o.index - o.buf = raw - o.index = 0 - - err = o.unmarshalType(p.stype, p.sprop, false, bas) - o.buf = obuf - o.index = oi - - return err -} - -// Decode a slice of embedded messages. -func (o *Buffer) dec_slice_struct_message(p *Properties, base structPointer) error { - return o.dec_slice_struct(p, false, base) -} - -// Decode a slice of embedded groups. -func (o *Buffer) dec_slice_struct_group(p *Properties, base structPointer) error { - return o.dec_slice_struct(p, true, base) -} - -// Decode a slice of structs ([]*struct). -func (o *Buffer) dec_slice_struct(p *Properties, is_group bool, base structPointer) error { - v := reflect.New(p.stype) - bas := toStructPointer(v) - structPointer_StructPointerSlice(base, p.field).Append(bas) - - if is_group { - err := o.unmarshalType(p.stype, p.sprop, is_group, bas) - return err - } - - raw, err := o.DecodeRawBytes(false) - if err != nil { + if u, ok := pb.(Unmarshaler); ok { + // NOTE: The history of proto have unfortunately been inconsistent + // whether Unmarshaler should or should not implicitly clear itself. + // Some implementations do, most do not. + // Thus, calling this here may or may not do what people want. + // + // See https://github.com/golang/protobuf/issues/424 + err := u.Unmarshal(p.buf[p.index:]) + p.index = len(p.buf) return err } - // If the object can unmarshal itself, let it. - if p.isUnmarshaler { - iv := v.Interface() - return iv.(Unmarshaler).Unmarshal(raw) - } - - obuf := o.buf - oi := o.index - o.buf = raw - o.index = 0 - - err = o.unmarshalType(p.stype, p.sprop, is_group, bas) - - o.buf = obuf - o.index = oi - + // Slow workaround for messages that aren't Unmarshalers. + // This includes some hand-coded .pb.go files and + // bootstrap protos. + // TODO: fix all of those and then add Unmarshal to + // the Message interface. Then: + // The cast above and code below can be deleted. + // The old unmarshaler can be deleted. + // Clients can call Unmarshal directly (can already do that, actually). + var info InternalMessageInfo + err := info.Unmarshal(pb, p.buf[p.index:]) + p.index = len(p.buf) return err } diff --git a/vendor/github.com/golang/protobuf/proto/discard.go b/vendor/github.com/golang/protobuf/proto/discard.go new file mode 100644 index 00000000000..dea2617ced3 --- /dev/null +++ b/vendor/github.com/golang/protobuf/proto/discard.go @@ -0,0 +1,350 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2017 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +import ( + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" +) + +type generatedDiscarder interface { + XXX_DiscardUnknown() +} + +// DiscardUnknown recursively discards all unknown fields from this message +// and all embedded messages. +// +// When unmarshaling a message with unrecognized fields, the tags and values +// of such fields are preserved in the Message. This allows a later call to +// marshal to be able to produce a message that continues to have those +// unrecognized fields. To avoid this, DiscardUnknown is used to +// explicitly clear the unknown fields after unmarshaling. +// +// For proto2 messages, the unknown fields of message extensions are only +// discarded from messages that have been accessed via GetExtension. +func DiscardUnknown(m Message) { + if m, ok := m.(generatedDiscarder); ok { + m.XXX_DiscardUnknown() + return + } + // TODO: Dynamically populate a InternalMessageInfo for legacy messages, + // but the master branch has no implementation for InternalMessageInfo, + // so it would be more work to replicate that approach. + discardLegacy(m) +} + +// DiscardUnknown recursively discards all unknown fields. +func (a *InternalMessageInfo) DiscardUnknown(m Message) { + di := atomicLoadDiscardInfo(&a.discard) + if di == nil { + di = getDiscardInfo(reflect.TypeOf(m).Elem()) + atomicStoreDiscardInfo(&a.discard, di) + } + di.discard(toPointer(&m)) +} + +type discardInfo struct { + typ reflect.Type + + initialized int32 // 0: only typ is valid, 1: everything is valid + lock sync.Mutex + + fields []discardFieldInfo + unrecognized field +} + +type discardFieldInfo struct { + field field // Offset of field, guaranteed to be valid + discard func(src pointer) +} + +var ( + discardInfoMap = map[reflect.Type]*discardInfo{} + discardInfoLock sync.Mutex +) + +func getDiscardInfo(t reflect.Type) *discardInfo { + discardInfoLock.Lock() + defer discardInfoLock.Unlock() + di := discardInfoMap[t] + if di == nil { + di = &discardInfo{typ: t} + discardInfoMap[t] = di + } + return di +} + +func (di *discardInfo) discard(src pointer) { + if src.isNil() { + return // Nothing to do. + } + + if atomic.LoadInt32(&di.initialized) == 0 { + di.computeDiscardInfo() + } + + for _, fi := range di.fields { + sfp := src.offset(fi.field) + fi.discard(sfp) + } + + // For proto2 messages, only discard unknown fields in message extensions + // that have been accessed via GetExtension. + if em, err := extendable(src.asPointerTo(di.typ).Interface()); err == nil { + // Ignore lock since DiscardUnknown is not concurrency safe. + emm, _ := em.extensionsRead() + for _, mx := range emm { + if m, ok := mx.value.(Message); ok { + DiscardUnknown(m) + } + } + } + + if di.unrecognized.IsValid() { + *src.offset(di.unrecognized).toBytes() = nil + } +} + +func (di *discardInfo) computeDiscardInfo() { + di.lock.Lock() + defer di.lock.Unlock() + if di.initialized != 0 { + return + } + t := di.typ + n := t.NumField() + + for i := 0; i < n; i++ { + f := t.Field(i) + if strings.HasPrefix(f.Name, "XXX_") { + continue + } + + dfi := discardFieldInfo{field: toField(&f)} + tf := f.Type + + // Unwrap tf to get its most basic type. + var isPointer, isSlice bool + if tf.Kind() == reflect.Slice && tf.Elem().Kind() != reflect.Uint8 { + isSlice = true + tf = tf.Elem() + } + if tf.Kind() == reflect.Ptr { + isPointer = true + tf = tf.Elem() + } + if isPointer && isSlice && tf.Kind() != reflect.Struct { + panic(fmt.Sprintf("%v.%s cannot be a slice of pointers to primitive types", t, f.Name)) + } + + switch tf.Kind() { + case reflect.Struct: + switch { + case !isPointer: + panic(fmt.Sprintf("%v.%s cannot be a direct struct value", t, f.Name)) + case isSlice: // E.g., []*pb.T + di := getDiscardInfo(tf) + dfi.discard = func(src pointer) { + sps := src.getPointerSlice() + for _, sp := range sps { + if !sp.isNil() { + di.discard(sp) + } + } + } + default: // E.g., *pb.T + di := getDiscardInfo(tf) + dfi.discard = func(src pointer) { + sp := src.getPointer() + if !sp.isNil() { + di.discard(sp) + } + } + } + case reflect.Map: + switch { + case isPointer || isSlice: + panic(fmt.Sprintf("%v.%s cannot be a pointer to a map or a slice of map values", t, f.Name)) + default: // E.g., map[K]V + if tf.Elem().Kind() == reflect.Ptr { // Proto struct (e.g., *T) + dfi.discard = func(src pointer) { + sm := src.asPointerTo(tf).Elem() + if sm.Len() == 0 { + return + } + for _, key := range sm.MapKeys() { + val := sm.MapIndex(key) + DiscardUnknown(val.Interface().(Message)) + } + } + } else { + dfi.discard = func(pointer) {} // Noop + } + } + case reflect.Interface: + // Must be oneof field. + switch { + case isPointer || isSlice: + panic(fmt.Sprintf("%v.%s cannot be a pointer to a interface or a slice of interface values", t, f.Name)) + default: // E.g., interface{} + // TODO: Make this faster? + dfi.discard = func(src pointer) { + su := src.asPointerTo(tf).Elem() + if !su.IsNil() { + sv := su.Elem().Elem().Field(0) + if sv.Kind() == reflect.Ptr && sv.IsNil() { + return + } + switch sv.Type().Kind() { + case reflect.Ptr: // Proto struct (e.g., *T) + DiscardUnknown(sv.Interface().(Message)) + } + } + } + } + default: + continue + } + di.fields = append(di.fields, dfi) + } + + di.unrecognized = invalidField + if f, ok := t.FieldByName("XXX_unrecognized"); ok { + if f.Type != reflect.TypeOf([]byte{}) { + panic("expected XXX_unrecognized to be of type []byte") + } + di.unrecognized = toField(&f) + } + + atomic.StoreInt32(&di.initialized, 1) +} + +func discardLegacy(m Message) { + v := reflect.ValueOf(m) + if v.Kind() != reflect.Ptr || v.IsNil() { + return + } + v = v.Elem() + if v.Kind() != reflect.Struct { + return + } + t := v.Type() + + for i := 0; i < v.NumField(); i++ { + f := t.Field(i) + if strings.HasPrefix(f.Name, "XXX_") { + continue + } + vf := v.Field(i) + tf := f.Type + + // Unwrap tf to get its most basic type. + var isPointer, isSlice bool + if tf.Kind() == reflect.Slice && tf.Elem().Kind() != reflect.Uint8 { + isSlice = true + tf = tf.Elem() + } + if tf.Kind() == reflect.Ptr { + isPointer = true + tf = tf.Elem() + } + if isPointer && isSlice && tf.Kind() != reflect.Struct { + panic(fmt.Sprintf("%T.%s cannot be a slice of pointers to primitive types", m, f.Name)) + } + + switch tf.Kind() { + case reflect.Struct: + switch { + case !isPointer: + panic(fmt.Sprintf("%T.%s cannot be a direct struct value", m, f.Name)) + case isSlice: // E.g., []*pb.T + for j := 0; j < vf.Len(); j++ { + discardLegacy(vf.Index(j).Interface().(Message)) + } + default: // E.g., *pb.T + discardLegacy(vf.Interface().(Message)) + } + case reflect.Map: + switch { + case isPointer || isSlice: + panic(fmt.Sprintf("%T.%s cannot be a pointer to a map or a slice of map values", m, f.Name)) + default: // E.g., map[K]V + tv := vf.Type().Elem() + if tv.Kind() == reflect.Ptr && tv.Implements(protoMessageType) { // Proto struct (e.g., *T) + for _, key := range vf.MapKeys() { + val := vf.MapIndex(key) + discardLegacy(val.Interface().(Message)) + } + } + } + case reflect.Interface: + // Must be oneof field. + switch { + case isPointer || isSlice: + panic(fmt.Sprintf("%T.%s cannot be a pointer to a interface or a slice of interface values", m, f.Name)) + default: // E.g., test_proto.isCommunique_Union interface + if !vf.IsNil() && f.Tag.Get("protobuf_oneof") != "" { + vf = vf.Elem() // E.g., *test_proto.Communique_Msg + if !vf.IsNil() { + vf = vf.Elem() // E.g., test_proto.Communique_Msg + vf = vf.Field(0) // E.g., Proto struct (e.g., *T) or primitive value + if vf.Kind() == reflect.Ptr { + discardLegacy(vf.Interface().(Message)) + } + } + } + } + } + } + + if vf := v.FieldByName("XXX_unrecognized"); vf.IsValid() { + if vf.Type() != reflect.TypeOf([]byte{}) { + panic("expected XXX_unrecognized to be of type []byte") + } + vf.Set(reflect.ValueOf([]byte(nil))) + } + + // For proto2 messages, only discard unknown fields in message extensions + // that have been accessed via GetExtension. + if em, err := extendable(m); err == nil { + // Ignore lock since discardLegacy is not concurrency safe. + emm, _ := em.extensionsRead() + for _, mx := range emm { + if m, ok := mx.value.(Message); ok { + discardLegacy(m) + } + } + } +} diff --git a/vendor/github.com/golang/protobuf/proto/encode.go b/vendor/github.com/golang/protobuf/proto/encode.go index 2b30f84626a..c27d35f866b 100644 --- a/vendor/github.com/golang/protobuf/proto/encode.go +++ b/vendor/github.com/golang/protobuf/proto/encode.go @@ -39,7 +39,6 @@ import ( "errors" "fmt" "reflect" - "sort" ) // RequiredNotSetError is the error returned if Marshal is called with @@ -82,10 +81,6 @@ var ( const maxVarintBytes = 10 // maximum length of a varint -// maxMarshalSize is the largest allowed size of an encoded protobuf, -// since C++ and Java use signed int32s for the size. -const maxMarshalSize = 1<<31 - 1 - // EncodeVarint returns the varint encoding of x. // This is the format for the // int32, int64, uint32, uint64, bool, and enum @@ -119,18 +114,27 @@ func (p *Buffer) EncodeVarint(x uint64) error { // SizeVarint returns the varint encoding size of an integer. func SizeVarint(x uint64) int { - return sizeVarint(x) -} - -func sizeVarint(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n + switch { + case x < 1<<7: + return 1 + case x < 1<<14: + return 2 + case x < 1<<21: + return 3 + case x < 1<<28: + return 4 + case x < 1<<35: + return 5 + case x < 1<<42: + return 6 + case x < 1<<49: + return 7 + case x < 1<<56: + return 8 + case x < 1<<63: + return 9 + } + return 10 } // EncodeFixed64 writes a 64-bit integer to the Buffer. @@ -149,10 +153,6 @@ func (p *Buffer) EncodeFixed64(x uint64) error { return nil } -func sizeFixed64(x uint64) int { - return 8 -} - // EncodeFixed32 writes a 32-bit integer to the Buffer. // This is the format for the // fixed32, sfixed32, and float protocol buffer types. @@ -165,10 +165,6 @@ func (p *Buffer) EncodeFixed32(x uint64) error { return nil } -func sizeFixed32(x uint64) int { - return 4 -} - // EncodeZigzag64 writes a zigzag-encoded 64-bit integer // to the Buffer. // This is the format used for the sint64 protocol buffer type. @@ -177,10 +173,6 @@ func (p *Buffer) EncodeZigzag64(x uint64) error { return p.EncodeVarint(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func sizeZigzag64(x uint64) int { - return sizeVarint(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} - // EncodeZigzag32 writes a zigzag-encoded 32-bit integer // to the Buffer. // This is the format used for the sint32 protocol buffer type. @@ -189,10 +181,6 @@ func (p *Buffer) EncodeZigzag32(x uint64) error { return p.EncodeVarint(uint64((uint32(x) << 1) ^ uint32((int32(x) >> 31)))) } -func sizeZigzag32(x uint64) int { - return sizeVarint(uint64((uint32(x) << 1) ^ uint32((int32(x) >> 31)))) -} - // EncodeRawBytes writes a count-delimited byte buffer to the Buffer. // This is the format used for the bytes protocol buffer // type and for embedded messages. @@ -202,11 +190,6 @@ func (p *Buffer) EncodeRawBytes(b []byte) error { return nil } -func sizeRawBytes(b []byte) int { - return sizeVarint(uint64(len(b))) + - len(b) -} - // EncodeStringBytes writes an encoded string to the Buffer. // This is the format used for the proto2 string type. func (p *Buffer) EncodeStringBytes(s string) error { @@ -215,319 +198,17 @@ func (p *Buffer) EncodeStringBytes(s string) error { return nil } -func sizeStringBytes(s string) int { - return sizeVarint(uint64(len(s))) + - len(s) -} - // Marshaler is the interface representing objects that can marshal themselves. type Marshaler interface { Marshal() ([]byte, error) } -// Marshal takes the protocol buffer -// and encodes it into the wire format, returning the data. -func Marshal(pb Message) ([]byte, error) { - // Can the object marshal itself? - if m, ok := pb.(Marshaler); ok { - return m.Marshal() - } - p := NewBuffer(nil) - err := p.Marshal(pb) - if p.buf == nil && err == nil { - // Return a non-nil slice on success. - return []byte{}, nil - } - return p.buf, err -} - // EncodeMessage writes the protocol buffer to the Buffer, // prefixed by a varint-encoded length. func (p *Buffer) EncodeMessage(pb Message) error { - t, base, err := getbase(pb) - if structPointer_IsNil(base) { - return ErrNil - } - if err == nil { - var state errorState - err = p.enc_len_struct(GetProperties(t.Elem()), base, &state) - } - return err -} - -// Marshal takes the protocol buffer -// and encodes it into the wire format, writing the result to the -// Buffer. -func (p *Buffer) Marshal(pb Message) error { - // Can the object marshal itself? - if m, ok := pb.(Marshaler); ok { - data, err := m.Marshal() - p.buf = append(p.buf, data...) - return err - } - - t, base, err := getbase(pb) - if structPointer_IsNil(base) { - return ErrNil - } - if err == nil { - err = p.enc_struct(GetProperties(t.Elem()), base) - } - - if collectStats { - (stats).Encode++ // Parens are to work around a goimports bug. - } - - if len(p.buf) > maxMarshalSize { - return ErrTooLarge - } - return err -} - -// Size returns the encoded size of a protocol buffer. -func Size(pb Message) (n int) { - // Can the object marshal itself? If so, Size is slow. - // TODO: add Size to Marshaler, or add a Sizer interface. - if m, ok := pb.(Marshaler); ok { - b, _ := m.Marshal() - return len(b) - } - - t, base, err := getbase(pb) - if structPointer_IsNil(base) { - return 0 - } - if err == nil { - n = size_struct(GetProperties(t.Elem()), base) - } - - if collectStats { - (stats).Size++ // Parens are to work around a goimports bug. - } - - return -} - -// Individual type encoders. - -// Encode a bool. -func (o *Buffer) enc_bool(p *Properties, base structPointer) error { - v := *structPointer_Bool(base, p.field) - if v == nil { - return ErrNil - } - x := 0 - if *v { - x = 1 - } - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, uint64(x)) - return nil -} - -func (o *Buffer) enc_proto3_bool(p *Properties, base structPointer) error { - v := *structPointer_BoolVal(base, p.field) - if !v { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, 1) - return nil -} - -func size_bool(p *Properties, base structPointer) int { - v := *structPointer_Bool(base, p.field) - if v == nil { - return 0 - } - return len(p.tagcode) + 1 // each bool takes exactly one byte -} - -func size_proto3_bool(p *Properties, base structPointer) int { - v := *structPointer_BoolVal(base, p.field) - if !v && !p.oneof { - return 0 - } - return len(p.tagcode) + 1 // each bool takes exactly one byte -} - -// Encode an int32. -func (o *Buffer) enc_int32(p *Properties, base structPointer) error { - v := structPointer_Word32(base, p.field) - if word32_IsNil(v) { - return ErrNil - } - x := int32(word32_Get(v)) // permit sign extension to use full 64-bit range - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, uint64(x)) - return nil -} - -func (o *Buffer) enc_proto3_int32(p *Properties, base structPointer) error { - v := structPointer_Word32Val(base, p.field) - x := int32(word32Val_Get(v)) // permit sign extension to use full 64-bit range - if x == 0 { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, uint64(x)) - return nil -} - -func size_int32(p *Properties, base structPointer) (n int) { - v := structPointer_Word32(base, p.field) - if word32_IsNil(v) { - return 0 - } - x := int32(word32_Get(v)) // permit sign extension to use full 64-bit range - n += len(p.tagcode) - n += p.valSize(uint64(x)) - return -} - -func size_proto3_int32(p *Properties, base structPointer) (n int) { - v := structPointer_Word32Val(base, p.field) - x := int32(word32Val_Get(v)) // permit sign extension to use full 64-bit range - if x == 0 && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += p.valSize(uint64(x)) - return -} - -// Encode a uint32. -// Exactly the same as int32, except for no sign extension. -func (o *Buffer) enc_uint32(p *Properties, base structPointer) error { - v := structPointer_Word32(base, p.field) - if word32_IsNil(v) { - return ErrNil - } - x := word32_Get(v) - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, uint64(x)) - return nil -} - -func (o *Buffer) enc_proto3_uint32(p *Properties, base structPointer) error { - v := structPointer_Word32Val(base, p.field) - x := word32Val_Get(v) - if x == 0 { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, uint64(x)) - return nil -} - -func size_uint32(p *Properties, base structPointer) (n int) { - v := structPointer_Word32(base, p.field) - if word32_IsNil(v) { - return 0 - } - x := word32_Get(v) - n += len(p.tagcode) - n += p.valSize(uint64(x)) - return -} - -func size_proto3_uint32(p *Properties, base structPointer) (n int) { - v := structPointer_Word32Val(base, p.field) - x := word32Val_Get(v) - if x == 0 && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += p.valSize(uint64(x)) - return -} - -// Encode an int64. -func (o *Buffer) enc_int64(p *Properties, base structPointer) error { - v := structPointer_Word64(base, p.field) - if word64_IsNil(v) { - return ErrNil - } - x := word64_Get(v) - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, x) - return nil -} - -func (o *Buffer) enc_proto3_int64(p *Properties, base structPointer) error { - v := structPointer_Word64Val(base, p.field) - x := word64Val_Get(v) - if x == 0 { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, x) - return nil -} - -func size_int64(p *Properties, base structPointer) (n int) { - v := structPointer_Word64(base, p.field) - if word64_IsNil(v) { - return 0 - } - x := word64_Get(v) - n += len(p.tagcode) - n += p.valSize(x) - return -} - -func size_proto3_int64(p *Properties, base structPointer) (n int) { - v := structPointer_Word64Val(base, p.field) - x := word64Val_Get(v) - if x == 0 && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += p.valSize(x) - return -} - -// Encode a string. -func (o *Buffer) enc_string(p *Properties, base structPointer) error { - v := *structPointer_String(base, p.field) - if v == nil { - return ErrNil - } - x := *v - o.buf = append(o.buf, p.tagcode...) - o.EncodeStringBytes(x) - return nil -} - -func (o *Buffer) enc_proto3_string(p *Properties, base structPointer) error { - v := *structPointer_StringVal(base, p.field) - if v == "" { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeStringBytes(v) - return nil -} - -func size_string(p *Properties, base structPointer) (n int) { - v := *structPointer_String(base, p.field) - if v == nil { - return 0 - } - x := *v - n += len(p.tagcode) - n += sizeStringBytes(x) - return -} - -func size_proto3_string(p *Properties, base structPointer) (n int) { - v := *structPointer_StringVal(base, p.field) - if v == "" && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += sizeStringBytes(v) - return + siz := Size(pb) + p.EncodeVarint(uint64(siz)) + return p.Marshal(pb) } // All protocol buffer fields are nillable, but be careful. @@ -538,825 +219,3 @@ func isNil(v reflect.Value) bool { } return false } - -// Encode a message struct. -func (o *Buffer) enc_struct_message(p *Properties, base structPointer) error { - var state errorState - structp := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(structp) { - return ErrNil - } - - // Can the object marshal itself? - if p.isMarshaler { - m := structPointer_Interface(structp, p.stype).(Marshaler) - data, err := m.Marshal() - if err != nil && !state.shouldContinue(err, nil) { - return err - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeRawBytes(data) - return state.err - } - - o.buf = append(o.buf, p.tagcode...) - return o.enc_len_struct(p.sprop, structp, &state) -} - -func size_struct_message(p *Properties, base structPointer) int { - structp := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(structp) { - return 0 - } - - // Can the object marshal itself? - if p.isMarshaler { - m := structPointer_Interface(structp, p.stype).(Marshaler) - data, _ := m.Marshal() - n0 := len(p.tagcode) - n1 := sizeRawBytes(data) - return n0 + n1 - } - - n0 := len(p.tagcode) - n1 := size_struct(p.sprop, structp) - n2 := sizeVarint(uint64(n1)) // size of encoded length - return n0 + n1 + n2 -} - -// Encode a group struct. -func (o *Buffer) enc_struct_group(p *Properties, base structPointer) error { - var state errorState - b := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(b) { - return ErrNil - } - - o.EncodeVarint(uint64((p.Tag << 3) | WireStartGroup)) - err := o.enc_struct(p.sprop, b) - if err != nil && !state.shouldContinue(err, nil) { - return err - } - o.EncodeVarint(uint64((p.Tag << 3) | WireEndGroup)) - return state.err -} - -func size_struct_group(p *Properties, base structPointer) (n int) { - b := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(b) { - return 0 - } - - n += sizeVarint(uint64((p.Tag << 3) | WireStartGroup)) - n += size_struct(p.sprop, b) - n += sizeVarint(uint64((p.Tag << 3) | WireEndGroup)) - return -} - -// Encode a slice of bools ([]bool). -func (o *Buffer) enc_slice_bool(p *Properties, base structPointer) error { - s := *structPointer_BoolSlice(base, p.field) - l := len(s) - if l == 0 { - return ErrNil - } - for _, x := range s { - o.buf = append(o.buf, p.tagcode...) - v := uint64(0) - if x { - v = 1 - } - p.valEnc(o, v) - } - return nil -} - -func size_slice_bool(p *Properties, base structPointer) int { - s := *structPointer_BoolSlice(base, p.field) - l := len(s) - if l == 0 { - return 0 - } - return l * (len(p.tagcode) + 1) // each bool takes exactly one byte -} - -// Encode a slice of bools ([]bool) in packed format. -func (o *Buffer) enc_slice_packed_bool(p *Properties, base structPointer) error { - s := *structPointer_BoolSlice(base, p.field) - l := len(s) - if l == 0 { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeVarint(uint64(l)) // each bool takes exactly one byte - for _, x := range s { - v := uint64(0) - if x { - v = 1 - } - p.valEnc(o, v) - } - return nil -} - -func size_slice_packed_bool(p *Properties, base structPointer) (n int) { - s := *structPointer_BoolSlice(base, p.field) - l := len(s) - if l == 0 { - return 0 - } - n += len(p.tagcode) - n += sizeVarint(uint64(l)) - n += l // each bool takes exactly one byte - return -} - -// Encode a slice of bytes ([]byte). -func (o *Buffer) enc_slice_byte(p *Properties, base structPointer) error { - s := *structPointer_Bytes(base, p.field) - if s == nil { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeRawBytes(s) - return nil -} - -func (o *Buffer) enc_proto3_slice_byte(p *Properties, base structPointer) error { - s := *structPointer_Bytes(base, p.field) - if len(s) == 0 { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeRawBytes(s) - return nil -} - -func size_slice_byte(p *Properties, base structPointer) (n int) { - s := *structPointer_Bytes(base, p.field) - if s == nil && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += sizeRawBytes(s) - return -} - -func size_proto3_slice_byte(p *Properties, base structPointer) (n int) { - s := *structPointer_Bytes(base, p.field) - if len(s) == 0 && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += sizeRawBytes(s) - return -} - -// Encode a slice of int32s ([]int32). -func (o *Buffer) enc_slice_int32(p *Properties, base structPointer) error { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - for i := 0; i < l; i++ { - o.buf = append(o.buf, p.tagcode...) - x := int32(s.Index(i)) // permit sign extension to use full 64-bit range - p.valEnc(o, uint64(x)) - } - return nil -} - -func size_slice_int32(p *Properties, base structPointer) (n int) { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - for i := 0; i < l; i++ { - n += len(p.tagcode) - x := int32(s.Index(i)) // permit sign extension to use full 64-bit range - n += p.valSize(uint64(x)) - } - return -} - -// Encode a slice of int32s ([]int32) in packed format. -func (o *Buffer) enc_slice_packed_int32(p *Properties, base structPointer) error { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - // TODO: Reuse a Buffer. - buf := NewBuffer(nil) - for i := 0; i < l; i++ { - x := int32(s.Index(i)) // permit sign extension to use full 64-bit range - p.valEnc(buf, uint64(x)) - } - - o.buf = append(o.buf, p.tagcode...) - o.EncodeVarint(uint64(len(buf.buf))) - o.buf = append(o.buf, buf.buf...) - return nil -} - -func size_slice_packed_int32(p *Properties, base structPointer) (n int) { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - var bufSize int - for i := 0; i < l; i++ { - x := int32(s.Index(i)) // permit sign extension to use full 64-bit range - bufSize += p.valSize(uint64(x)) - } - - n += len(p.tagcode) - n += sizeVarint(uint64(bufSize)) - n += bufSize - return -} - -// Encode a slice of uint32s ([]uint32). -// Exactly the same as int32, except for no sign extension. -func (o *Buffer) enc_slice_uint32(p *Properties, base structPointer) error { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - for i := 0; i < l; i++ { - o.buf = append(o.buf, p.tagcode...) - x := s.Index(i) - p.valEnc(o, uint64(x)) - } - return nil -} - -func size_slice_uint32(p *Properties, base structPointer) (n int) { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - for i := 0; i < l; i++ { - n += len(p.tagcode) - x := s.Index(i) - n += p.valSize(uint64(x)) - } - return -} - -// Encode a slice of uint32s ([]uint32) in packed format. -// Exactly the same as int32, except for no sign extension. -func (o *Buffer) enc_slice_packed_uint32(p *Properties, base structPointer) error { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - // TODO: Reuse a Buffer. - buf := NewBuffer(nil) - for i := 0; i < l; i++ { - p.valEnc(buf, uint64(s.Index(i))) - } - - o.buf = append(o.buf, p.tagcode...) - o.EncodeVarint(uint64(len(buf.buf))) - o.buf = append(o.buf, buf.buf...) - return nil -} - -func size_slice_packed_uint32(p *Properties, base structPointer) (n int) { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - var bufSize int - for i := 0; i < l; i++ { - bufSize += p.valSize(uint64(s.Index(i))) - } - - n += len(p.tagcode) - n += sizeVarint(uint64(bufSize)) - n += bufSize - return -} - -// Encode a slice of int64s ([]int64). -func (o *Buffer) enc_slice_int64(p *Properties, base structPointer) error { - s := structPointer_Word64Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - for i := 0; i < l; i++ { - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, s.Index(i)) - } - return nil -} - -func size_slice_int64(p *Properties, base structPointer) (n int) { - s := structPointer_Word64Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - for i := 0; i < l; i++ { - n += len(p.tagcode) - n += p.valSize(s.Index(i)) - } - return -} - -// Encode a slice of int64s ([]int64) in packed format. -func (o *Buffer) enc_slice_packed_int64(p *Properties, base structPointer) error { - s := structPointer_Word64Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - // TODO: Reuse a Buffer. - buf := NewBuffer(nil) - for i := 0; i < l; i++ { - p.valEnc(buf, s.Index(i)) - } - - o.buf = append(o.buf, p.tagcode...) - o.EncodeVarint(uint64(len(buf.buf))) - o.buf = append(o.buf, buf.buf...) - return nil -} - -func size_slice_packed_int64(p *Properties, base structPointer) (n int) { - s := structPointer_Word64Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - var bufSize int - for i := 0; i < l; i++ { - bufSize += p.valSize(s.Index(i)) - } - - n += len(p.tagcode) - n += sizeVarint(uint64(bufSize)) - n += bufSize - return -} - -// Encode a slice of slice of bytes ([][]byte). -func (o *Buffer) enc_slice_slice_byte(p *Properties, base structPointer) error { - ss := *structPointer_BytesSlice(base, p.field) - l := len(ss) - if l == 0 { - return ErrNil - } - for i := 0; i < l; i++ { - o.buf = append(o.buf, p.tagcode...) - o.EncodeRawBytes(ss[i]) - } - return nil -} - -func size_slice_slice_byte(p *Properties, base structPointer) (n int) { - ss := *structPointer_BytesSlice(base, p.field) - l := len(ss) - if l == 0 { - return 0 - } - n += l * len(p.tagcode) - for i := 0; i < l; i++ { - n += sizeRawBytes(ss[i]) - } - return -} - -// Encode a slice of strings ([]string). -func (o *Buffer) enc_slice_string(p *Properties, base structPointer) error { - ss := *structPointer_StringSlice(base, p.field) - l := len(ss) - for i := 0; i < l; i++ { - o.buf = append(o.buf, p.tagcode...) - o.EncodeStringBytes(ss[i]) - } - return nil -} - -func size_slice_string(p *Properties, base structPointer) (n int) { - ss := *structPointer_StringSlice(base, p.field) - l := len(ss) - n += l * len(p.tagcode) - for i := 0; i < l; i++ { - n += sizeStringBytes(ss[i]) - } - return -} - -// Encode a slice of message structs ([]*struct). -func (o *Buffer) enc_slice_struct_message(p *Properties, base structPointer) error { - var state errorState - s := structPointer_StructPointerSlice(base, p.field) - l := s.Len() - - for i := 0; i < l; i++ { - structp := s.Index(i) - if structPointer_IsNil(structp) { - return errRepeatedHasNil - } - - // Can the object marshal itself? - if p.isMarshaler { - m := structPointer_Interface(structp, p.stype).(Marshaler) - data, err := m.Marshal() - if err != nil && !state.shouldContinue(err, nil) { - return err - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeRawBytes(data) - continue - } - - o.buf = append(o.buf, p.tagcode...) - err := o.enc_len_struct(p.sprop, structp, &state) - if err != nil && !state.shouldContinue(err, nil) { - if err == ErrNil { - return errRepeatedHasNil - } - return err - } - } - return state.err -} - -func size_slice_struct_message(p *Properties, base structPointer) (n int) { - s := structPointer_StructPointerSlice(base, p.field) - l := s.Len() - n += l * len(p.tagcode) - for i := 0; i < l; i++ { - structp := s.Index(i) - if structPointer_IsNil(structp) { - return // return the size up to this point - } - - // Can the object marshal itself? - if p.isMarshaler { - m := structPointer_Interface(structp, p.stype).(Marshaler) - data, _ := m.Marshal() - n += sizeRawBytes(data) - continue - } - - n0 := size_struct(p.sprop, structp) - n1 := sizeVarint(uint64(n0)) // size of encoded length - n += n0 + n1 - } - return -} - -// Encode a slice of group structs ([]*struct). -func (o *Buffer) enc_slice_struct_group(p *Properties, base structPointer) error { - var state errorState - s := structPointer_StructPointerSlice(base, p.field) - l := s.Len() - - for i := 0; i < l; i++ { - b := s.Index(i) - if structPointer_IsNil(b) { - return errRepeatedHasNil - } - - o.EncodeVarint(uint64((p.Tag << 3) | WireStartGroup)) - - err := o.enc_struct(p.sprop, b) - - if err != nil && !state.shouldContinue(err, nil) { - if err == ErrNil { - return errRepeatedHasNil - } - return err - } - - o.EncodeVarint(uint64((p.Tag << 3) | WireEndGroup)) - } - return state.err -} - -func size_slice_struct_group(p *Properties, base structPointer) (n int) { - s := structPointer_StructPointerSlice(base, p.field) - l := s.Len() - - n += l * sizeVarint(uint64((p.Tag<<3)|WireStartGroup)) - n += l * sizeVarint(uint64((p.Tag<<3)|WireEndGroup)) - for i := 0; i < l; i++ { - b := s.Index(i) - if structPointer_IsNil(b) { - return // return size up to this point - } - - n += size_struct(p.sprop, b) - } - return -} - -// Encode an extension map. -func (o *Buffer) enc_map(p *Properties, base structPointer) error { - exts := structPointer_ExtMap(base, p.field) - if err := encodeExtensionsMap(*exts); err != nil { - return err - } - - return o.enc_map_body(*exts) -} - -func (o *Buffer) enc_exts(p *Properties, base structPointer) error { - exts := structPointer_Extensions(base, p.field) - - v, mu := exts.extensionsRead() - if v == nil { - return nil - } - - mu.Lock() - defer mu.Unlock() - if err := encodeExtensionsMap(v); err != nil { - return err - } - - return o.enc_map_body(v) -} - -func (o *Buffer) enc_map_body(v map[int32]Extension) error { - // Fast-path for common cases: zero or one extensions. - if len(v) <= 1 { - for _, e := range v { - o.buf = append(o.buf, e.enc...) - } - return nil - } - - // Sort keys to provide a deterministic encoding. - keys := make([]int, 0, len(v)) - for k := range v { - keys = append(keys, int(k)) - } - sort.Ints(keys) - - for _, k := range keys { - o.buf = append(o.buf, v[int32(k)].enc...) - } - return nil -} - -func size_map(p *Properties, base structPointer) int { - v := structPointer_ExtMap(base, p.field) - return extensionsMapSize(*v) -} - -func size_exts(p *Properties, base structPointer) int { - v := structPointer_Extensions(base, p.field) - return extensionsSize(v) -} - -// Encode a map field. -func (o *Buffer) enc_new_map(p *Properties, base structPointer) error { - var state errorState // XXX: or do we need to plumb this through? - - /* - A map defined as - map map_field = N; - is encoded in the same way as - message MapFieldEntry { - key_type key = 1; - value_type value = 2; - } - repeated MapFieldEntry map_field = N; - */ - - v := structPointer_NewAt(base, p.field, p.mtype).Elem() // map[K]V - if v.Len() == 0 { - return nil - } - - keycopy, valcopy, keybase, valbase := mapEncodeScratch(p.mtype) - - enc := func() error { - if err := p.mkeyprop.enc(o, p.mkeyprop, keybase); err != nil { - return err - } - if err := p.mvalprop.enc(o, p.mvalprop, valbase); err != nil && err != ErrNil { - return err - } - return nil - } - - // Don't sort map keys. It is not required by the spec, and C++ doesn't do it. - for _, key := range v.MapKeys() { - val := v.MapIndex(key) - - keycopy.Set(key) - valcopy.Set(val) - - o.buf = append(o.buf, p.tagcode...) - if err := o.enc_len_thing(enc, &state); err != nil { - return err - } - } - return nil -} - -func size_new_map(p *Properties, base structPointer) int { - v := structPointer_NewAt(base, p.field, p.mtype).Elem() // map[K]V - - keycopy, valcopy, keybase, valbase := mapEncodeScratch(p.mtype) - - n := 0 - for _, key := range v.MapKeys() { - val := v.MapIndex(key) - keycopy.Set(key) - valcopy.Set(val) - - // Tag codes for key and val are the responsibility of the sub-sizer. - keysize := p.mkeyprop.size(p.mkeyprop, keybase) - valsize := p.mvalprop.size(p.mvalprop, valbase) - entry := keysize + valsize - // Add on tag code and length of map entry itself. - n += len(p.tagcode) + sizeVarint(uint64(entry)) + entry - } - return n -} - -// mapEncodeScratch returns a new reflect.Value matching the map's value type, -// and a structPointer suitable for passing to an encoder or sizer. -func mapEncodeScratch(mapType reflect.Type) (keycopy, valcopy reflect.Value, keybase, valbase structPointer) { - // Prepare addressable doubly-indirect placeholders for the key and value types. - // This is needed because the element-type encoders expect **T, but the map iteration produces T. - - keycopy = reflect.New(mapType.Key()).Elem() // addressable K - keyptr := reflect.New(reflect.PtrTo(keycopy.Type())).Elem() // addressable *K - keyptr.Set(keycopy.Addr()) // - keybase = toStructPointer(keyptr.Addr()) // **K - - // Value types are more varied and require special handling. - switch mapType.Elem().Kind() { - case reflect.Slice: - // []byte - var dummy []byte - valcopy = reflect.ValueOf(&dummy).Elem() // addressable []byte - valbase = toStructPointer(valcopy.Addr()) - case reflect.Ptr: - // message; the generated field type is map[K]*Msg (so V is *Msg), - // so we only need one level of indirection. - valcopy = reflect.New(mapType.Elem()).Elem() // addressable V - valbase = toStructPointer(valcopy.Addr()) - default: - // everything else - valcopy = reflect.New(mapType.Elem()).Elem() // addressable V - valptr := reflect.New(reflect.PtrTo(valcopy.Type())).Elem() // addressable *V - valptr.Set(valcopy.Addr()) // - valbase = toStructPointer(valptr.Addr()) // **V - } - return -} - -// Encode a struct. -func (o *Buffer) enc_struct(prop *StructProperties, base structPointer) error { - var state errorState - // Encode fields in tag order so that decoders may use optimizations - // that depend on the ordering. - // https://developers.google.com/protocol-buffers/docs/encoding#order - for _, i := range prop.order { - p := prop.Prop[i] - if p.enc != nil { - err := p.enc(o, p, base) - if err != nil { - if err == ErrNil { - if p.Required && state.err == nil { - state.err = &RequiredNotSetError{p.Name} - } - } else if err == errRepeatedHasNil { - // Give more context to nil values in repeated fields. - return errors.New("repeated field " + p.OrigName + " has nil element") - } else if !state.shouldContinue(err, p) { - return err - } - } - if len(o.buf) > maxMarshalSize { - return ErrTooLarge - } - } - } - - // Do oneof fields. - if prop.oneofMarshaler != nil { - m := structPointer_Interface(base, prop.stype).(Message) - if err := prop.oneofMarshaler(m, o); err == ErrNil { - return errOneofHasNil - } else if err != nil { - return err - } - } - - // Add unrecognized fields at the end. - if prop.unrecField.IsValid() { - v := *structPointer_Bytes(base, prop.unrecField) - if len(o.buf)+len(v) > maxMarshalSize { - return ErrTooLarge - } - if len(v) > 0 { - o.buf = append(o.buf, v...) - } - } - - return state.err -} - -func size_struct(prop *StructProperties, base structPointer) (n int) { - for _, i := range prop.order { - p := prop.Prop[i] - if p.size != nil { - n += p.size(p, base) - } - } - - // Add unrecognized fields at the end. - if prop.unrecField.IsValid() { - v := *structPointer_Bytes(base, prop.unrecField) - n += len(v) - } - - // Factor in any oneof fields. - if prop.oneofSizer != nil { - m := structPointer_Interface(base, prop.stype).(Message) - n += prop.oneofSizer(m) - } - - return -} - -var zeroes [20]byte // longer than any conceivable sizeVarint - -// Encode a struct, preceded by its encoded length (as a varint). -func (o *Buffer) enc_len_struct(prop *StructProperties, base structPointer, state *errorState) error { - return o.enc_len_thing(func() error { return o.enc_struct(prop, base) }, state) -} - -// Encode something, preceded by its encoded length (as a varint). -func (o *Buffer) enc_len_thing(enc func() error, state *errorState) error { - iLen := len(o.buf) - o.buf = append(o.buf, 0, 0, 0, 0) // reserve four bytes for length - iMsg := len(o.buf) - err := enc() - if err != nil && !state.shouldContinue(err, nil) { - return err - } - lMsg := len(o.buf) - iMsg - lLen := sizeVarint(uint64(lMsg)) - switch x := lLen - (iMsg - iLen); { - case x > 0: // actual length is x bytes larger than the space we reserved - // Move msg x bytes right. - o.buf = append(o.buf, zeroes[:x]...) - copy(o.buf[iMsg+x:], o.buf[iMsg:iMsg+lMsg]) - case x < 0: // actual length is x bytes smaller than the space we reserved - // Move msg x bytes left. - copy(o.buf[iMsg+x:], o.buf[iMsg:iMsg+lMsg]) - o.buf = o.buf[:len(o.buf)+x] // x is negative - } - // Encode the length in the reserved space. - o.buf = o.buf[:iLen] - o.EncodeVarint(uint64(lMsg)) - o.buf = o.buf[:len(o.buf)+lMsg] - return state.err -} - -// errorState maintains the first error that occurs and updates that error -// with additional context. -type errorState struct { - err error -} - -// shouldContinue reports whether encoding should continue upon encountering the -// given error. If the error is RequiredNotSetError, shouldContinue returns true -// and, if this is the first appearance of that error, remembers it for future -// reporting. -// -// If prop is not nil, it may update any error with additional context about the -// field with the error. -func (s *errorState) shouldContinue(err error, prop *Properties) bool { - // Ignore unset required fields. - reqNotSet, ok := err.(*RequiredNotSetError) - if !ok { - return false - } - if s.err == nil { - if prop != nil { - err = &RequiredNotSetError{prop.Name + "." + reqNotSet.field} - } - s.err = err - } - return true -} diff --git a/vendor/github.com/golang/protobuf/proto/equal.go b/vendor/github.com/golang/protobuf/proto/equal.go index 2ed1cf59666..d4db5a1c145 100644 --- a/vendor/github.com/golang/protobuf/proto/equal.go +++ b/vendor/github.com/golang/protobuf/proto/equal.go @@ -109,15 +109,6 @@ func equalStruct(v1, v2 reflect.Value) bool { // set/unset mismatch return false } - b1, ok := f1.Interface().(raw) - if ok { - b2 := f2.Interface().(raw) - // RawMessage - if !bytes.Equal(b1.Bytes(), b2.Bytes()) { - return false - } - continue - } f1, f2 = f1.Elem(), f2.Elem() } if !equalAny(f1, f2, sprop.Prop[i]) { @@ -146,11 +137,7 @@ func equalStruct(v1, v2 reflect.Value) bool { u1 := uf.Bytes() u2 := v2.FieldByName("XXX_unrecognized").Bytes() - if !bytes.Equal(u1, u2) { - return false - } - - return true + return bytes.Equal(u1, u2) } // v1 and v2 are known to have the same type. @@ -261,6 +248,15 @@ func equalExtMap(base reflect.Type, em1, em2 map[int32]Extension) bool { m1, m2 := e1.value, e2.value + if m1 == nil && m2 == nil { + // Both have only encoded form. + if bytes.Equal(e1.enc, e2.enc) { + continue + } + // The bytes are different, but the extensions might still be + // equal. We need to decode them to compare. + } + if m1 != nil && m2 != nil { // Both are unencoded. if !equalAny(reflect.ValueOf(m1), reflect.ValueOf(m2), nil) { @@ -276,8 +272,12 @@ func equalExtMap(base reflect.Type, em1, em2 map[int32]Extension) bool { desc = m[extNum] } if desc == nil { + // If both have only encoded form and the bytes are the same, + // it is handled above. We get here when the bytes are different. + // We don't know how to decode it, so just compare them as byte + // slices. log.Printf("proto: don't know how to compare extension %d of %v", extNum, base) - continue + return false } var err error if m1 == nil { diff --git a/vendor/github.com/golang/protobuf/proto/extensions.go b/vendor/github.com/golang/protobuf/proto/extensions.go index eaad2183126..816a3b9d6c0 100644 --- a/vendor/github.com/golang/protobuf/proto/extensions.go +++ b/vendor/github.com/golang/protobuf/proto/extensions.go @@ -38,6 +38,7 @@ package proto import ( "errors" "fmt" + "io" "reflect" "strconv" "sync" @@ -91,14 +92,29 @@ func (n notLocker) Unlock() {} // extendable returns the extendableProto interface for the given generated proto message. // If the proto message has the old extension format, it returns a wrapper that implements // the extendableProto interface. -func extendable(p interface{}) (extendableProto, bool) { - if ep, ok := p.(extendableProto); ok { - return ep, ok - } - if ep, ok := p.(extendableProtoV1); ok { - return extensionAdapter{ep}, ok +func extendable(p interface{}) (extendableProto, error) { + switch p := p.(type) { + case extendableProto: + if isNilPtr(p) { + return nil, fmt.Errorf("proto: nil %T is not extendable", p) + } + return p, nil + case extendableProtoV1: + if isNilPtr(p) { + return nil, fmt.Errorf("proto: nil %T is not extendable", p) + } + return extensionAdapter{p}, nil } - return nil, false + // Don't allocate a specific error containing %T: + // this is the hot path for Clone and MarshalText. + return nil, errNotExtendable +} + +var errNotExtendable = errors.New("proto: not an extendable proto.Message") + +func isNilPtr(x interface{}) bool { + v := reflect.ValueOf(x) + return v.Kind() == reflect.Ptr && v.IsNil() } // XXX_InternalExtensions is an internal representation of proto extensions. @@ -143,9 +159,6 @@ func (e *XXX_InternalExtensions) extensionsRead() (map[int32]Extension, sync.Loc return e.p.extensionMap, &e.p.mu } -var extendableProtoType = reflect.TypeOf((*extendableProto)(nil)).Elem() -var extendableProtoV1Type = reflect.TypeOf((*extendableProtoV1)(nil)).Elem() - // ExtensionDesc represents an extension specification. // Used in generated code from the protocol compiler. type ExtensionDesc struct { @@ -179,8 +192,8 @@ type Extension struct { // SetRawExtension is for testing only. func SetRawExtension(base Message, id int32, b []byte) { - epb, ok := extendable(base) - if !ok { + epb, err := extendable(base) + if err != nil { return } extmap := epb.extensionsWrite() @@ -205,7 +218,7 @@ func checkExtensionTypes(pb extendableProto, extension *ExtensionDesc) error { pbi = ea.extendableProtoV1 } if a, b := reflect.TypeOf(pbi), reflect.TypeOf(extension.ExtendedType); a != b { - return errors.New("proto: bad extended type; " + b.String() + " does not extend " + a.String()) + return fmt.Errorf("proto: bad extended type; %v does not extend %v", b, a) } // Check the range. if !isExtensionField(pb, extension.Field) { @@ -250,85 +263,11 @@ func extensionProperties(ed *ExtensionDesc) *Properties { return prop } -// encode encodes any unmarshaled (unencoded) extensions in e. -func encodeExtensions(e *XXX_InternalExtensions) error { - m, mu := e.extensionsRead() - if m == nil { - return nil // fast path - } - mu.Lock() - defer mu.Unlock() - return encodeExtensionsMap(m) -} - -// encode encodes any unmarshaled (unencoded) extensions in e. -func encodeExtensionsMap(m map[int32]Extension) error { - for k, e := range m { - if e.value == nil || e.desc == nil { - // Extension is only in its encoded form. - continue - } - - // We don't skip extensions that have an encoded form set, - // because the extension value may have been mutated after - // the last time this function was called. - - et := reflect.TypeOf(e.desc.ExtensionType) - props := extensionProperties(e.desc) - - p := NewBuffer(nil) - // If e.value has type T, the encoder expects a *struct{ X T }. - // Pass a *T with a zero field and hope it all works out. - x := reflect.New(et) - x.Elem().Set(reflect.ValueOf(e.value)) - if err := props.enc(p, props, toStructPointer(x)); err != nil { - return err - } - e.enc = p.buf - m[k] = e - } - return nil -} - -func extensionsSize(e *XXX_InternalExtensions) (n int) { - m, mu := e.extensionsRead() - if m == nil { - return 0 - } - mu.Lock() - defer mu.Unlock() - return extensionsMapSize(m) -} - -func extensionsMapSize(m map[int32]Extension) (n int) { - for _, e := range m { - if e.value == nil || e.desc == nil { - // Extension is only in its encoded form. - n += len(e.enc) - continue - } - - // We don't skip extensions that have an encoded form set, - // because the extension value may have been mutated after - // the last time this function was called. - - et := reflect.TypeOf(e.desc.ExtensionType) - props := extensionProperties(e.desc) - - // If e.value has type T, the encoder expects a *struct{ X T }. - // Pass a *T with a zero field and hope it all works out. - x := reflect.New(et) - x.Elem().Set(reflect.ValueOf(e.value)) - n += props.size(props, toStructPointer(x)) - } - return -} - // HasExtension returns whether the given extension is present in pb. func HasExtension(pb Message, extension *ExtensionDesc) bool { // TODO: Check types, field numbers, etc.? - epb, ok := extendable(pb) - if !ok { + epb, err := extendable(pb) + if err != nil { return false } extmap, mu := epb.extensionsRead() @@ -336,15 +275,15 @@ func HasExtension(pb Message, extension *ExtensionDesc) bool { return false } mu.Lock() - _, ok = extmap[extension.Field] + _, ok := extmap[extension.Field] mu.Unlock() return ok } // ClearExtension removes the given extension from pb. func ClearExtension(pb Message, extension *ExtensionDesc) { - epb, ok := extendable(pb) - if !ok { + epb, err := extendable(pb) + if err != nil { return } // TODO: Check types, field numbers, etc.? @@ -352,16 +291,26 @@ func ClearExtension(pb Message, extension *ExtensionDesc) { delete(extmap, extension.Field) } -// GetExtension parses and returns the given extension of pb. -// If the extension is not present and has no default value it returns ErrMissingExtension. +// GetExtension retrieves a proto2 extended field from pb. +// +// If the descriptor is type complete (i.e., ExtensionDesc.ExtensionType is non-nil), +// then GetExtension parses the encoded field and returns a Go value of the specified type. +// If the field is not present, then the default value is returned (if one is specified), +// otherwise ErrMissingExtension is reported. +// +// If the descriptor is not type complete (i.e., ExtensionDesc.ExtensionType is nil), +// then GetExtension returns the raw encoded bytes of the field extension. func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) { - epb, ok := extendable(pb) - if !ok { - return nil, errors.New("proto: not an extendable proto") + epb, err := extendable(pb) + if err != nil { + return nil, err } - if err := checkExtensionTypes(epb, extension); err != nil { - return nil, err + if extension.ExtendedType != nil { + // can only check type if this is a complete descriptor + if err := checkExtensionTypes(epb, extension); err != nil { + return nil, err + } } emap, mu := epb.extensionsRead() @@ -388,6 +337,11 @@ func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) { return e.value, nil } + if extension.ExtensionType == nil { + // incomplete descriptor + return e.enc, nil + } + v, err := decodeExtension(e.enc, extension) if err != nil { return nil, err @@ -405,6 +359,11 @@ func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) { // defaultExtensionValue returns the default value for extension. // If no default for an extension is defined ErrMissingExtension is returned. func defaultExtensionValue(extension *ExtensionDesc) (interface{}, error) { + if extension.ExtensionType == nil { + // incomplete descriptor, so no default + return nil, ErrMissingExtension + } + t := reflect.TypeOf(extension.ExtensionType) props := extensionProperties(extension) @@ -439,31 +398,28 @@ func defaultExtensionValue(extension *ExtensionDesc) (interface{}, error) { // decodeExtension decodes an extension encoded in b. func decodeExtension(b []byte, extension *ExtensionDesc) (interface{}, error) { - o := NewBuffer(b) - t := reflect.TypeOf(extension.ExtensionType) - - props := extensionProperties(extension) + unmarshal := typeUnmarshaler(t, extension.Tag) // t is a pointer to a struct, pointer to basic type or a slice. - // Allocate a "field" to store the pointer/slice itself; the - // pointer/slice will be stored here. We pass - // the address of this field to props.dec. - // This passes a zero field and a *t and lets props.dec - // interpret it as a *struct{ x t }. + // Allocate space to store the pointer/slice. value := reflect.New(t).Elem() + var err error for { - // Discard wire type and field number varint. It isn't needed. - if _, err := o.DecodeVarint(); err != nil { - return nil, err + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF } + b = b[n:] + wire := int(x) & 7 - if err := props.dec(o, props, toStructPointer(value.Addr())); err != nil { + b, err = unmarshal(b, valToPointer(value.Addr()), wire) + if err != nil { return nil, err } - if o.index >= len(o.buf) { + if len(b) == 0 { break } } @@ -473,9 +429,9 @@ func decodeExtension(b []byte, extension *ExtensionDesc) (interface{}, error) { // GetExtensions returns a slice of the extensions present in pb that are also listed in es. // The returned slice has the same length as es; missing extensions will appear as nil elements. func GetExtensions(pb Message, es []*ExtensionDesc) (extensions []interface{}, err error) { - epb, ok := extendable(pb) - if !ok { - return nil, errors.New("proto: not an extendable proto") + epb, err := extendable(pb) + if err != nil { + return nil, err } extensions = make([]interface{}, len(es)) for i, e := range es { @@ -494,9 +450,9 @@ func GetExtensions(pb Message, es []*ExtensionDesc) (extensions []interface{}, e // For non-registered extensions, ExtensionDescs returns an incomplete descriptor containing // just the Field field, which defines the extension's field number. func ExtensionDescs(pb Message) ([]*ExtensionDesc, error) { - epb, ok := extendable(pb) - if !ok { - return nil, fmt.Errorf("proto: %T is not an extendable proto.Message", pb) + epb, err := extendable(pb) + if err != nil { + return nil, err } registeredExtensions := RegisteredExtensions(pb) @@ -523,9 +479,9 @@ func ExtensionDescs(pb Message) ([]*ExtensionDesc, error) { // SetExtension sets the specified extension of pb to the specified value. func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error { - epb, ok := extendable(pb) - if !ok { - return errors.New("proto: not an extendable proto") + epb, err := extendable(pb) + if err != nil { + return err } if err := checkExtensionTypes(epb, extension); err != nil { return err @@ -550,8 +506,8 @@ func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error // ClearAllExtensions clears all extensions from pb. func ClearAllExtensions(pb Message) { - epb, ok := extendable(pb) - if !ok { + epb, err := extendable(pb) + if err != nil { return } m := epb.extensionsWrite() diff --git a/vendor/github.com/golang/protobuf/proto/lib.go b/vendor/github.com/golang/protobuf/proto/lib.go index ac4ddbc0759..0e2191b8ada 100644 --- a/vendor/github.com/golang/protobuf/proto/lib.go +++ b/vendor/github.com/golang/protobuf/proto/lib.go @@ -73,7 +73,6 @@ for a protocol buffer variable v: When the .proto file specifies `syntax="proto3"`, there are some differences: - Non-repeated fields of non-message type are values instead of pointers. - - Getters are only generated for message and oneof fields. - Enum types do not get an Enum method. The simplest way to describe this is to see an example. @@ -266,6 +265,7 @@ package proto import ( "encoding/json" + "errors" "fmt" "log" "reflect" @@ -274,6 +274,8 @@ import ( "sync" ) +var errInvalidUTF8 = errors.New("proto: invalid UTF-8 string") + // Message is implemented by generated protocol buffer messages. type Message interface { Reset() @@ -310,16 +312,7 @@ type Buffer struct { buf []byte // encode/decode byte stream index int // read point - // pools of basic types to amortize allocation. - bools []bool - uint32s []uint32 - uint64s []uint64 - - // extra pools, only used with pointer_reflect.go - int32s []int32 - int64s []int64 - float32s []float32 - float64s []float64 + deterministic bool } // NewBuffer allocates a new Buffer and initializes its internal data to @@ -344,6 +337,30 @@ func (p *Buffer) SetBuf(s []byte) { // Bytes returns the contents of the Buffer. func (p *Buffer) Bytes() []byte { return p.buf } +// SetDeterministic sets whether to use deterministic serialization. +// +// Deterministic serialization guarantees that for a given binary, equal +// messages will always be serialized to the same bytes. This implies: +// +// - Repeated serialization of a message will return the same bytes. +// - Different processes of the same binary (which may be executing on +// different machines) will serialize equal messages to the same bytes. +// +// Note that the deterministic serialization is NOT canonical across +// languages. It is not guaranteed to remain stable over time. It is unstable +// across different builds with schema changes due to unknown fields. +// Users who need canonical serialization (e.g., persistent storage in a +// canonical form, fingerprinting, etc.) should define their own +// canonicalization specification and implement their own serializer rather +// than relying on this API. +// +// If deterministic serialization is requested, map entries will be sorted +// by keys in lexographical order. This is an implementation detail and +// subject to change. +func (p *Buffer) SetDeterministic(deterministic bool) { + p.deterministic = deterministic +} + /* * Helper routines for simplifying the creation of optional fields of basic type. */ @@ -832,22 +849,12 @@ func fieldDefault(ft reflect.Type, prop *Properties) (sf *scalarField, nestedMes return sf, false, nil } +// mapKeys returns a sort.Interface to be used for sorting the map keys. // Map fields may have key types of non-float scalars, strings and enums. -// The easiest way to sort them in some deterministic order is to use fmt. -// If this turns out to be inefficient we can always consider other options, -// such as doing a Schwartzian transform. - func mapKeys(vs []reflect.Value) sort.Interface { - s := mapKeySorter{ - vs: vs, - // default Less function: textual comparison - less: func(a, b reflect.Value) bool { - return fmt.Sprint(a.Interface()) < fmt.Sprint(b.Interface()) - }, - } + s := mapKeySorter{vs: vs} - // Type specialization per https://developers.google.com/protocol-buffers/docs/proto#maps; - // numeric keys are sorted numerically. + // Type specialization per https://developers.google.com/protocol-buffers/docs/proto#maps. if len(vs) == 0 { return s } @@ -856,6 +863,12 @@ func mapKeys(vs []reflect.Value) sort.Interface { s.less = func(a, b reflect.Value) bool { return a.Int() < b.Int() } case reflect.Uint32, reflect.Uint64: s.less = func(a, b reflect.Value) bool { return a.Uint() < b.Uint() } + case reflect.Bool: + s.less = func(a, b reflect.Value) bool { return !a.Bool() && b.Bool() } // false < true + case reflect.String: + s.less = func(a, b reflect.Value) bool { return a.String() < b.String() } + default: + panic(fmt.Sprintf("unsupported map key type: %v", vs[0].Kind())) } return s @@ -896,3 +909,13 @@ const ProtoPackageIsVersion2 = true // ProtoPackageIsVersion1 is referenced from generated protocol buffer files // to assert that that code is compatible with this version of the proto package. const ProtoPackageIsVersion1 = true + +// InternalMessageInfo is a type used internally by generated .pb.go files. +// This type is not intended to be used by non-generated code. +// This type is not subject to any compatibility guarantee. +type InternalMessageInfo struct { + marshal *marshalInfo + unmarshal *unmarshalInfo + merge *mergeInfo + discard *discardInfo +} diff --git a/vendor/github.com/golang/protobuf/proto/message_set.go b/vendor/github.com/golang/protobuf/proto/message_set.go index fd982decd66..3b6ca41d5e5 100644 --- a/vendor/github.com/golang/protobuf/proto/message_set.go +++ b/vendor/github.com/golang/protobuf/proto/message_set.go @@ -42,6 +42,7 @@ import ( "fmt" "reflect" "sort" + "sync" ) // errNoMessageTypeID occurs when a protocol buffer does not have a message type ID. @@ -94,10 +95,7 @@ func (ms *messageSet) find(pb Message) *_MessageSet_Item { } func (ms *messageSet) Has(pb Message) bool { - if ms.find(pb) != nil { - return true - } - return false + return ms.find(pb) != nil } func (ms *messageSet) Unmarshal(pb Message) error { @@ -150,46 +148,42 @@ func skipVarint(buf []byte) []byte { // MarshalMessageSet encodes the extension map represented by m in the message set wire format. // It is called by generated Marshal methods on protocol buffer messages with the message_set_wire_format option. func MarshalMessageSet(exts interface{}) ([]byte, error) { - var m map[int32]Extension + return marshalMessageSet(exts, false) +} + +// marshaMessageSet implements above function, with the opt to turn on / off deterministic during Marshal. +func marshalMessageSet(exts interface{}, deterministic bool) ([]byte, error) { switch exts := exts.(type) { case *XXX_InternalExtensions: - if err := encodeExtensions(exts); err != nil { - return nil, err - } - m, _ = exts.extensionsRead() + var u marshalInfo + siz := u.sizeMessageSet(exts) + b := make([]byte, 0, siz) + return u.appendMessageSet(b, exts, deterministic) + case map[int32]Extension: - if err := encodeExtensionsMap(exts); err != nil { - return nil, err + // This is an old-style extension map. + // Wrap it in a new-style XXX_InternalExtensions. + ie := XXX_InternalExtensions{ + p: &struct { + mu sync.Mutex + extensionMap map[int32]Extension + }{ + extensionMap: exts, + }, } - m = exts + + var u marshalInfo + siz := u.sizeMessageSet(&ie) + b := make([]byte, 0, siz) + return u.appendMessageSet(b, &ie, deterministic) + default: return nil, errors.New("proto: not an extension map") } - - // Sort extension IDs to provide a deterministic encoding. - // See also enc_map in encode.go. - ids := make([]int, 0, len(m)) - for id := range m { - ids = append(ids, int(id)) - } - sort.Ints(ids) - - ms := &messageSet{Item: make([]*_MessageSet_Item, 0, len(m))} - for _, id := range ids { - e := m[int32(id)] - // Remove the wire type and field number varint, as well as the length varint. - msg := skipVarint(skipVarint(e.enc)) - - ms.Item = append(ms.Item, &_MessageSet_Item{ - TypeId: Int32(int32(id)), - Message: msg, - }) - } - return Marshal(ms) } // UnmarshalMessageSet decodes the extension map encoded in buf in the message set wire format. -// It is called by generated Unmarshal methods on protocol buffer messages with the message_set_wire_format option. +// It is called by Unmarshal methods on protocol buffer messages with the message_set_wire_format option. func UnmarshalMessageSet(buf []byte, exts interface{}) error { var m map[int32]Extension switch exts := exts.(type) { @@ -235,7 +229,15 @@ func MarshalMessageSetJSON(exts interface{}) ([]byte, error) { var m map[int32]Extension switch exts := exts.(type) { case *XXX_InternalExtensions: - m, _ = exts.extensionsRead() + var mu sync.Locker + m, mu = exts.extensionsRead() + if m != nil { + // Keep the extensions map locked until we're done marshaling to prevent + // races between marshaling and unmarshaling the lazily-{en,de}coded + // values. + mu.Lock() + defer mu.Unlock() + } case map[int32]Extension: m = exts default: @@ -253,15 +255,16 @@ func MarshalMessageSetJSON(exts interface{}) ([]byte, error) { for i, id := range ids { ext := m[id] - if i > 0 { - b.WriteByte(',') - } - msd, ok := messageSetMap[id] if !ok { // Unknown type; we can't render it, so skip it. continue } + + if i > 0 && b.Len() > 1 { + b.WriteByte(',') + } + fmt.Fprintf(&b, `"[%s]":`, msd.name) x := ext.value diff --git a/vendor/github.com/golang/protobuf/proto/pointer_reflect.go b/vendor/github.com/golang/protobuf/proto/pointer_reflect.go index fb512e2e16d..b6cad90834b 100644 --- a/vendor/github.com/golang/protobuf/proto/pointer_reflect.go +++ b/vendor/github.com/golang/protobuf/proto/pointer_reflect.go @@ -29,7 +29,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +build appengine js +// +build purego appengine js // This file contains an implementation of proto field accesses using package reflect. // It is slower than the code in pointer_unsafe.go but it avoids package unsafe and can @@ -38,32 +38,13 @@ package proto import ( - "math" "reflect" + "sync" ) -// A structPointer is a pointer to a struct. -type structPointer struct { - v reflect.Value -} - -// toStructPointer returns a structPointer equivalent to the given reflect value. -// The reflect value must itself be a pointer to a struct. -func toStructPointer(v reflect.Value) structPointer { - return structPointer{v} -} - -// IsNil reports whether p is nil. -func structPointer_IsNil(p structPointer) bool { - return p.v.IsNil() -} +const unsafeAllowed = false -// Interface returns the struct pointer as an interface value. -func structPointer_Interface(p structPointer, _ reflect.Type) interface{} { - return p.v.Interface() -} - -// A field identifies a field in a struct, accessible from a structPointer. +// A field identifies a field in a struct, accessible from a pointer. // In this implementation, a field is identified by the sequence of field indices // passed to reflect's FieldByIndex. type field []int @@ -76,409 +57,301 @@ func toField(f *reflect.StructField) field { // invalidField is an invalid field identifier. var invalidField = field(nil) +// zeroField is a noop when calling pointer.offset. +var zeroField = field([]int{}) + // IsValid reports whether the field identifier is valid. func (f field) IsValid() bool { return f != nil } -// field returns the given field in the struct as a reflect value. -func structPointer_field(p structPointer, f field) reflect.Value { - // Special case: an extension map entry with a value of type T - // passes a *T to the struct-handling code with a zero field, - // expecting that it will be treated as equivalent to *struct{ X T }, - // which has the same memory layout. We have to handle that case - // specially, because reflect will panic if we call FieldByIndex on a - // non-struct. - if f == nil { - return p.v.Elem() - } - - return p.v.Elem().FieldByIndex(f) +// The pointer type is for the table-driven decoder. +// The implementation here uses a reflect.Value of pointer type to +// create a generic pointer. In pointer_unsafe.go we use unsafe +// instead of reflect to implement the same (but faster) interface. +type pointer struct { + v reflect.Value } -// ifield returns the given field in the struct as an interface value. -func structPointer_ifield(p structPointer, f field) interface{} { - return structPointer_field(p, f).Addr().Interface() +// toPointer converts an interface of pointer type to a pointer +// that points to the same target. +func toPointer(i *Message) pointer { + return pointer{v: reflect.ValueOf(*i)} } -// Bytes returns the address of a []byte field in the struct. -func structPointer_Bytes(p structPointer, f field) *[]byte { - return structPointer_ifield(p, f).(*[]byte) +// toAddrPointer converts an interface to a pointer that points to +// the interface data. +func toAddrPointer(i *interface{}, isptr bool) pointer { + v := reflect.ValueOf(*i) + u := reflect.New(v.Type()) + u.Elem().Set(v) + return pointer{v: u} } -// BytesSlice returns the address of a [][]byte field in the struct. -func structPointer_BytesSlice(p structPointer, f field) *[][]byte { - return structPointer_ifield(p, f).(*[][]byte) +// valToPointer converts v to a pointer. v must be of pointer type. +func valToPointer(v reflect.Value) pointer { + return pointer{v: v} } -// Bool returns the address of a *bool field in the struct. -func structPointer_Bool(p structPointer, f field) **bool { - return structPointer_ifield(p, f).(**bool) +// offset converts from a pointer to a structure to a pointer to +// one of its fields. +func (p pointer) offset(f field) pointer { + return pointer{v: p.v.Elem().FieldByIndex(f).Addr()} } -// BoolVal returns the address of a bool field in the struct. -func structPointer_BoolVal(p structPointer, f field) *bool { - return structPointer_ifield(p, f).(*bool) +func (p pointer) isNil() bool { + return p.v.IsNil() } -// BoolSlice returns the address of a []bool field in the struct. -func structPointer_BoolSlice(p structPointer, f field) *[]bool { - return structPointer_ifield(p, f).(*[]bool) +// grow updates the slice s in place to make it one element longer. +// s must be addressable. +// Returns the (addressable) new element. +func grow(s reflect.Value) reflect.Value { + n, m := s.Len(), s.Cap() + if n < m { + s.SetLen(n + 1) + } else { + s.Set(reflect.Append(s, reflect.Zero(s.Type().Elem()))) + } + return s.Index(n) } -// String returns the address of a *string field in the struct. -func structPointer_String(p structPointer, f field) **string { - return structPointer_ifield(p, f).(**string) +func (p pointer) toInt64() *int64 { + return p.v.Interface().(*int64) } - -// StringVal returns the address of a string field in the struct. -func structPointer_StringVal(p structPointer, f field) *string { - return structPointer_ifield(p, f).(*string) +func (p pointer) toInt64Ptr() **int64 { + return p.v.Interface().(**int64) } - -// StringSlice returns the address of a []string field in the struct. -func structPointer_StringSlice(p structPointer, f field) *[]string { - return structPointer_ifield(p, f).(*[]string) +func (p pointer) toInt64Slice() *[]int64 { + return p.v.Interface().(*[]int64) } -// Extensions returns the address of an extension map field in the struct. -func structPointer_Extensions(p structPointer, f field) *XXX_InternalExtensions { - return structPointer_ifield(p, f).(*XXX_InternalExtensions) -} +var int32ptr = reflect.TypeOf((*int32)(nil)) -// ExtMap returns the address of an extension map field in the struct. -func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension { - return structPointer_ifield(p, f).(*map[int32]Extension) +func (p pointer) toInt32() *int32 { + return p.v.Convert(int32ptr).Interface().(*int32) } -// NewAt returns the reflect.Value for a pointer to a field in the struct. -func structPointer_NewAt(p structPointer, f field, typ reflect.Type) reflect.Value { - return structPointer_field(p, f).Addr() +// The toInt32Ptr/Slice methods don't work because of enums. +// Instead, we must use set/get methods for the int32ptr/slice case. +/* + func (p pointer) toInt32Ptr() **int32 { + return p.v.Interface().(**int32) } - -// SetStructPointer writes a *struct field in the struct. -func structPointer_SetStructPointer(p structPointer, f field, q structPointer) { - structPointer_field(p, f).Set(q.v) + func (p pointer) toInt32Slice() *[]int32 { + return p.v.Interface().(*[]int32) } - -// GetStructPointer reads a *struct field in the struct. -func structPointer_GetStructPointer(p structPointer, f field) structPointer { - return structPointer{structPointer_field(p, f)} +*/ +func (p pointer) getInt32Ptr() *int32 { + if p.v.Type().Elem().Elem() == reflect.TypeOf(int32(0)) { + // raw int32 type + return p.v.Elem().Interface().(*int32) + } + // an enum + return p.v.Elem().Convert(int32PtrType).Interface().(*int32) +} +func (p pointer) setInt32Ptr(v int32) { + // Allocate value in a *int32. Possibly convert that to a *enum. + // Then assign it to a **int32 or **enum. + // Note: we can convert *int32 to *enum, but we can't convert + // **int32 to **enum! + p.v.Elem().Set(reflect.ValueOf(&v).Convert(p.v.Type().Elem())) +} + +// getInt32Slice copies []int32 from p as a new slice. +// This behavior differs from the implementation in pointer_unsafe.go. +func (p pointer) getInt32Slice() []int32 { + if p.v.Type().Elem().Elem() == reflect.TypeOf(int32(0)) { + // raw int32 type + return p.v.Elem().Interface().([]int32) + } + // an enum + // Allocate a []int32, then assign []enum's values into it. + // Note: we can't convert []enum to []int32. + slice := p.v.Elem() + s := make([]int32, slice.Len()) + for i := 0; i < slice.Len(); i++ { + s[i] = int32(slice.Index(i).Int()) + } + return s } -// StructPointerSlice the address of a []*struct field in the struct. -func structPointer_StructPointerSlice(p structPointer, f field) structPointerSlice { - return structPointerSlice{structPointer_field(p, f)} +// setInt32Slice copies []int32 into p as a new slice. +// This behavior differs from the implementation in pointer_unsafe.go. +func (p pointer) setInt32Slice(v []int32) { + if p.v.Type().Elem().Elem() == reflect.TypeOf(int32(0)) { + // raw int32 type + p.v.Elem().Set(reflect.ValueOf(v)) + return + } + // an enum + // Allocate a []enum, then assign []int32's values into it. + // Note: we can't convert []enum to []int32. + slice := reflect.MakeSlice(p.v.Type().Elem(), len(v), cap(v)) + for i, x := range v { + slice.Index(i).SetInt(int64(x)) + } + p.v.Elem().Set(slice) } - -// A structPointerSlice represents the address of a slice of pointers to structs -// (themselves messages or groups). That is, v.Type() is *[]*struct{...}. -type structPointerSlice struct { - v reflect.Value +func (p pointer) appendInt32Slice(v int32) { + grow(p.v.Elem()).SetInt(int64(v)) } -func (p structPointerSlice) Len() int { return p.v.Len() } -func (p structPointerSlice) Index(i int) structPointer { return structPointer{p.v.Index(i)} } -func (p structPointerSlice) Append(q structPointer) { - p.v.Set(reflect.Append(p.v, q.v)) +func (p pointer) toUint64() *uint64 { + return p.v.Interface().(*uint64) } - -var ( - int32Type = reflect.TypeOf(int32(0)) - uint32Type = reflect.TypeOf(uint32(0)) - float32Type = reflect.TypeOf(float32(0)) - int64Type = reflect.TypeOf(int64(0)) - uint64Type = reflect.TypeOf(uint64(0)) - float64Type = reflect.TypeOf(float64(0)) -) - -// A word32 represents a field of type *int32, *uint32, *float32, or *enum. -// That is, v.Type() is *int32, *uint32, *float32, or *enum and v is assignable. -type word32 struct { - v reflect.Value +func (p pointer) toUint64Ptr() **uint64 { + return p.v.Interface().(**uint64) } - -// IsNil reports whether p is nil. -func word32_IsNil(p word32) bool { - return p.v.IsNil() +func (p pointer) toUint64Slice() *[]uint64 { + return p.v.Interface().(*[]uint64) } - -// Set sets p to point at a newly allocated word with bits set to x. -func word32_Set(p word32, o *Buffer, x uint32) { - t := p.v.Type().Elem() - switch t { - case int32Type: - if len(o.int32s) == 0 { - o.int32s = make([]int32, uint32PoolSize) - } - o.int32s[0] = int32(x) - p.v.Set(reflect.ValueOf(&o.int32s[0])) - o.int32s = o.int32s[1:] - return - case uint32Type: - if len(o.uint32s) == 0 { - o.uint32s = make([]uint32, uint32PoolSize) - } - o.uint32s[0] = x - p.v.Set(reflect.ValueOf(&o.uint32s[0])) - o.uint32s = o.uint32s[1:] - return - case float32Type: - if len(o.float32s) == 0 { - o.float32s = make([]float32, uint32PoolSize) - } - o.float32s[0] = math.Float32frombits(x) - p.v.Set(reflect.ValueOf(&o.float32s[0])) - o.float32s = o.float32s[1:] - return - } - - // must be enum - p.v.Set(reflect.New(t)) - p.v.Elem().SetInt(int64(int32(x))) +func (p pointer) toUint32() *uint32 { + return p.v.Interface().(*uint32) } - -// Get gets the bits pointed at by p, as a uint32. -func word32_Get(p word32) uint32 { - elem := p.v.Elem() - switch elem.Kind() { - case reflect.Int32: - return uint32(elem.Int()) - case reflect.Uint32: - return uint32(elem.Uint()) - case reflect.Float32: - return math.Float32bits(float32(elem.Float())) - } - panic("unreachable") +func (p pointer) toUint32Ptr() **uint32 { + return p.v.Interface().(**uint32) } - -// Word32 returns a reference to a *int32, *uint32, *float32, or *enum field in the struct. -func structPointer_Word32(p structPointer, f field) word32 { - return word32{structPointer_field(p, f)} +func (p pointer) toUint32Slice() *[]uint32 { + return p.v.Interface().(*[]uint32) } - -// A word32Val represents a field of type int32, uint32, float32, or enum. -// That is, v.Type() is int32, uint32, float32, or enum and v is assignable. -type word32Val struct { - v reflect.Value +func (p pointer) toBool() *bool { + return p.v.Interface().(*bool) } - -// Set sets *p to x. -func word32Val_Set(p word32Val, x uint32) { - switch p.v.Type() { - case int32Type: - p.v.SetInt(int64(x)) - return - case uint32Type: - p.v.SetUint(uint64(x)) - return - case float32Type: - p.v.SetFloat(float64(math.Float32frombits(x))) - return - } - - // must be enum - p.v.SetInt(int64(int32(x))) +func (p pointer) toBoolPtr() **bool { + return p.v.Interface().(**bool) } - -// Get gets the bits pointed at by p, as a uint32. -func word32Val_Get(p word32Val) uint32 { - elem := p.v - switch elem.Kind() { - case reflect.Int32: - return uint32(elem.Int()) - case reflect.Uint32: - return uint32(elem.Uint()) - case reflect.Float32: - return math.Float32bits(float32(elem.Float())) - } - panic("unreachable") +func (p pointer) toBoolSlice() *[]bool { + return p.v.Interface().(*[]bool) } - -// Word32Val returns a reference to a int32, uint32, float32, or enum field in the struct. -func structPointer_Word32Val(p structPointer, f field) word32Val { - return word32Val{structPointer_field(p, f)} +func (p pointer) toFloat64() *float64 { + return p.v.Interface().(*float64) } - -// A word32Slice is a slice of 32-bit values. -// That is, v.Type() is []int32, []uint32, []float32, or []enum. -type word32Slice struct { - v reflect.Value +func (p pointer) toFloat64Ptr() **float64 { + return p.v.Interface().(**float64) } - -func (p word32Slice) Append(x uint32) { - n, m := p.v.Len(), p.v.Cap() - if n < m { - p.v.SetLen(n + 1) - } else { - t := p.v.Type().Elem() - p.v.Set(reflect.Append(p.v, reflect.Zero(t))) - } - elem := p.v.Index(n) - switch elem.Kind() { - case reflect.Int32: - elem.SetInt(int64(int32(x))) - case reflect.Uint32: - elem.SetUint(uint64(x)) - case reflect.Float32: - elem.SetFloat(float64(math.Float32frombits(x))) - } +func (p pointer) toFloat64Slice() *[]float64 { + return p.v.Interface().(*[]float64) } - -func (p word32Slice) Len() int { - return p.v.Len() +func (p pointer) toFloat32() *float32 { + return p.v.Interface().(*float32) } - -func (p word32Slice) Index(i int) uint32 { - elem := p.v.Index(i) - switch elem.Kind() { - case reflect.Int32: - return uint32(elem.Int()) - case reflect.Uint32: - return uint32(elem.Uint()) - case reflect.Float32: - return math.Float32bits(float32(elem.Float())) - } - panic("unreachable") +func (p pointer) toFloat32Ptr() **float32 { + return p.v.Interface().(**float32) } - -// Word32Slice returns a reference to a []int32, []uint32, []float32, or []enum field in the struct. -func structPointer_Word32Slice(p structPointer, f field) word32Slice { - return word32Slice{structPointer_field(p, f)} +func (p pointer) toFloat32Slice() *[]float32 { + return p.v.Interface().(*[]float32) } - -// word64 is like word32 but for 64-bit values. -type word64 struct { - v reflect.Value +func (p pointer) toString() *string { + return p.v.Interface().(*string) } - -func word64_Set(p word64, o *Buffer, x uint64) { - t := p.v.Type().Elem() - switch t { - case int64Type: - if len(o.int64s) == 0 { - o.int64s = make([]int64, uint64PoolSize) - } - o.int64s[0] = int64(x) - p.v.Set(reflect.ValueOf(&o.int64s[0])) - o.int64s = o.int64s[1:] - return - case uint64Type: - if len(o.uint64s) == 0 { - o.uint64s = make([]uint64, uint64PoolSize) - } - o.uint64s[0] = x - p.v.Set(reflect.ValueOf(&o.uint64s[0])) - o.uint64s = o.uint64s[1:] - return - case float64Type: - if len(o.float64s) == 0 { - o.float64s = make([]float64, uint64PoolSize) - } - o.float64s[0] = math.Float64frombits(x) - p.v.Set(reflect.ValueOf(&o.float64s[0])) - o.float64s = o.float64s[1:] - return - } - panic("unreachable") +func (p pointer) toStringPtr() **string { + return p.v.Interface().(**string) } - -func word64_IsNil(p word64) bool { - return p.v.IsNil() +func (p pointer) toStringSlice() *[]string { + return p.v.Interface().(*[]string) } - -func word64_Get(p word64) uint64 { - elem := p.v.Elem() - switch elem.Kind() { - case reflect.Int64: - return uint64(elem.Int()) - case reflect.Uint64: - return elem.Uint() - case reflect.Float64: - return math.Float64bits(elem.Float()) - } - panic("unreachable") +func (p pointer) toBytes() *[]byte { + return p.v.Interface().(*[]byte) } - -func structPointer_Word64(p structPointer, f field) word64 { - return word64{structPointer_field(p, f)} +func (p pointer) toBytesSlice() *[][]byte { + return p.v.Interface().(*[][]byte) +} +func (p pointer) toExtensions() *XXX_InternalExtensions { + return p.v.Interface().(*XXX_InternalExtensions) +} +func (p pointer) toOldExtensions() *map[int32]Extension { + return p.v.Interface().(*map[int32]Extension) +} +func (p pointer) getPointer() pointer { + return pointer{v: p.v.Elem()} +} +func (p pointer) setPointer(q pointer) { + p.v.Elem().Set(q.v) +} +func (p pointer) appendPointer(q pointer) { + grow(p.v.Elem()).Set(q.v) } -// word64Val is like word32Val but for 64-bit values. -type word64Val struct { - v reflect.Value +// getPointerSlice copies []*T from p as a new []pointer. +// This behavior differs from the implementation in pointer_unsafe.go. +func (p pointer) getPointerSlice() []pointer { + if p.v.IsNil() { + return nil + } + n := p.v.Elem().Len() + s := make([]pointer, n) + for i := 0; i < n; i++ { + s[i] = pointer{v: p.v.Elem().Index(i)} + } + return s } -func word64Val_Set(p word64Val, o *Buffer, x uint64) { - switch p.v.Type() { - case int64Type: - p.v.SetInt(int64(x)) - return - case uint64Type: - p.v.SetUint(x) - return - case float64Type: - p.v.SetFloat(math.Float64frombits(x)) +// setPointerSlice copies []pointer into p as a new []*T. +// This behavior differs from the implementation in pointer_unsafe.go. +func (p pointer) setPointerSlice(v []pointer) { + if v == nil { + p.v.Elem().Set(reflect.New(p.v.Elem().Type()).Elem()) return } - panic("unreachable") + s := reflect.MakeSlice(p.v.Elem().Type(), 0, len(v)) + for _, p := range v { + s = reflect.Append(s, p.v) + } + p.v.Elem().Set(s) } -func word64Val_Get(p word64Val) uint64 { - elem := p.v - switch elem.Kind() { - case reflect.Int64: - return uint64(elem.Int()) - case reflect.Uint64: - return elem.Uint() - case reflect.Float64: - return math.Float64bits(elem.Float()) +// getInterfacePointer returns a pointer that points to the +// interface data of the interface pointed by p. +func (p pointer) getInterfacePointer() pointer { + if p.v.Elem().IsNil() { + return pointer{v: p.v.Elem()} } - panic("unreachable") + return pointer{v: p.v.Elem().Elem().Elem().Field(0).Addr()} // *interface -> interface -> *struct -> struct } -func structPointer_Word64Val(p structPointer, f field) word64Val { - return word64Val{structPointer_field(p, f)} +func (p pointer) asPointerTo(t reflect.Type) reflect.Value { + // TODO: check that p.v.Type().Elem() == t? + return p.v } -type word64Slice struct { - v reflect.Value +func atomicLoadUnmarshalInfo(p **unmarshalInfo) *unmarshalInfo { + atomicLock.Lock() + defer atomicLock.Unlock() + return *p } - -func (p word64Slice) Append(x uint64) { - n, m := p.v.Len(), p.v.Cap() - if n < m { - p.v.SetLen(n + 1) - } else { - t := p.v.Type().Elem() - p.v.Set(reflect.Append(p.v, reflect.Zero(t))) - } - elem := p.v.Index(n) - switch elem.Kind() { - case reflect.Int64: - elem.SetInt(int64(int64(x))) - case reflect.Uint64: - elem.SetUint(uint64(x)) - case reflect.Float64: - elem.SetFloat(float64(math.Float64frombits(x))) - } +func atomicStoreUnmarshalInfo(p **unmarshalInfo, v *unmarshalInfo) { + atomicLock.Lock() + defer atomicLock.Unlock() + *p = v } - -func (p word64Slice) Len() int { - return p.v.Len() +func atomicLoadMarshalInfo(p **marshalInfo) *marshalInfo { + atomicLock.Lock() + defer atomicLock.Unlock() + return *p } - -func (p word64Slice) Index(i int) uint64 { - elem := p.v.Index(i) - switch elem.Kind() { - case reflect.Int64: - return uint64(elem.Int()) - case reflect.Uint64: - return uint64(elem.Uint()) - case reflect.Float64: - return math.Float64bits(float64(elem.Float())) - } - panic("unreachable") +func atomicStoreMarshalInfo(p **marshalInfo, v *marshalInfo) { + atomicLock.Lock() + defer atomicLock.Unlock() + *p = v } - -func structPointer_Word64Slice(p structPointer, f field) word64Slice { - return word64Slice{structPointer_field(p, f)} +func atomicLoadMergeInfo(p **mergeInfo) *mergeInfo { + atomicLock.Lock() + defer atomicLock.Unlock() + return *p +} +func atomicStoreMergeInfo(p **mergeInfo, v *mergeInfo) { + atomicLock.Lock() + defer atomicLock.Unlock() + *p = v } +func atomicLoadDiscardInfo(p **discardInfo) *discardInfo { + atomicLock.Lock() + defer atomicLock.Unlock() + return *p +} +func atomicStoreDiscardInfo(p **discardInfo, v *discardInfo) { + atomicLock.Lock() + defer atomicLock.Unlock() + *p = v +} + +var atomicLock sync.Mutex diff --git a/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go b/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go index 6b5567d47cd..d55a335d945 100644 --- a/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go +++ b/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go @@ -29,7 +29,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +build !appengine,!js +// +build !purego,!appengine,!js // This file contains the implementation of the proto field accesses using package unsafe. @@ -37,38 +37,13 @@ package proto import ( "reflect" + "sync/atomic" "unsafe" ) -// NOTE: These type_Foo functions would more idiomatically be methods, -// but Go does not allow methods on pointer types, and we must preserve -// some pointer type for the garbage collector. We use these -// funcs with clunky names as our poor approximation to methods. -// -// An alternative would be -// type structPointer struct { p unsafe.Pointer } -// but that does not registerize as well. - -// A structPointer is a pointer to a struct. -type structPointer unsafe.Pointer - -// toStructPointer returns a structPointer equivalent to the given reflect value. -func toStructPointer(v reflect.Value) structPointer { - return structPointer(unsafe.Pointer(v.Pointer())) -} - -// IsNil reports whether p is nil. -func structPointer_IsNil(p structPointer) bool { - return p == nil -} - -// Interface returns the struct pointer, assumed to have element type t, -// as an interface value. -func structPointer_Interface(p structPointer, t reflect.Type) interface{} { - return reflect.NewAt(t, unsafe.Pointer(p)).Interface() -} +const unsafeAllowed = true -// A field identifies a field in a struct, accessible from a structPointer. +// A field identifies a field in a struct, accessible from a pointer. // In this implementation, a field is identified by its byte offset from the start of the struct. type field uintptr @@ -80,191 +55,254 @@ func toField(f *reflect.StructField) field { // invalidField is an invalid field identifier. const invalidField = ^field(0) +// zeroField is a noop when calling pointer.offset. +const zeroField = field(0) + // IsValid reports whether the field identifier is valid. func (f field) IsValid() bool { - return f != ^field(0) + return f != invalidField } -// Bytes returns the address of a []byte field in the struct. -func structPointer_Bytes(p structPointer, f field) *[]byte { - return (*[]byte)(unsafe.Pointer(uintptr(p) + uintptr(f))) +// The pointer type below is for the new table-driven encoder/decoder. +// The implementation here uses unsafe.Pointer to create a generic pointer. +// In pointer_reflect.go we use reflect instead of unsafe to implement +// the same (but slower) interface. +type pointer struct { + p unsafe.Pointer } -// BytesSlice returns the address of a [][]byte field in the struct. -func structPointer_BytesSlice(p structPointer, f field) *[][]byte { - return (*[][]byte)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} +// size of pointer +var ptrSize = unsafe.Sizeof(uintptr(0)) -// Bool returns the address of a *bool field in the struct. -func structPointer_Bool(p structPointer, f field) **bool { - return (**bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) +// toPointer converts an interface of pointer type to a pointer +// that points to the same target. +func toPointer(i *Message) pointer { + // Super-tricky - read pointer out of data word of interface value. + // Saves ~25ns over the equivalent: + // return valToPointer(reflect.ValueOf(*i)) + return pointer{p: (*[2]unsafe.Pointer)(unsafe.Pointer(i))[1]} } -// BoolVal returns the address of a bool field in the struct. -func structPointer_BoolVal(p structPointer, f field) *bool { - return (*bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) +// toAddrPointer converts an interface to a pointer that points to +// the interface data. +func toAddrPointer(i *interface{}, isptr bool) pointer { + // Super-tricky - read or get the address of data word of interface value. + if isptr { + // The interface is of pointer type, thus it is a direct interface. + // The data word is the pointer data itself. We take its address. + return pointer{p: unsafe.Pointer(uintptr(unsafe.Pointer(i)) + ptrSize)} + } + // The interface is not of pointer type. The data word is the pointer + // to the data. + return pointer{p: (*[2]unsafe.Pointer)(unsafe.Pointer(i))[1]} } -// BoolSlice returns the address of a []bool field in the struct. -func structPointer_BoolSlice(p structPointer, f field) *[]bool { - return (*[]bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) +// valToPointer converts v to a pointer. v must be of pointer type. +func valToPointer(v reflect.Value) pointer { + return pointer{p: unsafe.Pointer(v.Pointer())} } -// String returns the address of a *string field in the struct. -func structPointer_String(p structPointer, f field) **string { - return (**string)(unsafe.Pointer(uintptr(p) + uintptr(f))) +// offset converts from a pointer to a structure to a pointer to +// one of its fields. +func (p pointer) offset(f field) pointer { + // For safety, we should panic if !f.IsValid, however calling panic causes + // this to no longer be inlineable, which is a serious performance cost. + /* + if !f.IsValid() { + panic("invalid field") + } + */ + return pointer{p: unsafe.Pointer(uintptr(p.p) + uintptr(f))} } -// StringVal returns the address of a string field in the struct. -func structPointer_StringVal(p structPointer, f field) *string { - return (*string)(unsafe.Pointer(uintptr(p) + uintptr(f))) +func (p pointer) isNil() bool { + return p.p == nil } -// StringSlice returns the address of a []string field in the struct. -func structPointer_StringSlice(p structPointer, f field) *[]string { - return (*[]string)(unsafe.Pointer(uintptr(p) + uintptr(f))) +func (p pointer) toInt64() *int64 { + return (*int64)(p.p) } - -// ExtMap returns the address of an extension map field in the struct. -func structPointer_Extensions(p structPointer, f field) *XXX_InternalExtensions { - return (*XXX_InternalExtensions)(unsafe.Pointer(uintptr(p) + uintptr(f))) +func (p pointer) toInt64Ptr() **int64 { + return (**int64)(p.p) } - -func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension { - return (*map[int32]Extension)(unsafe.Pointer(uintptr(p) + uintptr(f))) +func (p pointer) toInt64Slice() *[]int64 { + return (*[]int64)(p.p) } - -// NewAt returns the reflect.Value for a pointer to a field in the struct. -func structPointer_NewAt(p structPointer, f field, typ reflect.Type) reflect.Value { - return reflect.NewAt(typ, unsafe.Pointer(uintptr(p)+uintptr(f))) +func (p pointer) toInt32() *int32 { + return (*int32)(p.p) } -// SetStructPointer writes a *struct field in the struct. -func structPointer_SetStructPointer(p structPointer, f field, q structPointer) { - *(*structPointer)(unsafe.Pointer(uintptr(p) + uintptr(f))) = q +// See pointer_reflect.go for why toInt32Ptr/Slice doesn't exist. +/* + func (p pointer) toInt32Ptr() **int32 { + return (**int32)(p.p) + } + func (p pointer) toInt32Slice() *[]int32 { + return (*[]int32)(p.p) + } +*/ +func (p pointer) getInt32Ptr() *int32 { + return *(**int32)(p.p) } - -// GetStructPointer reads a *struct field in the struct. -func structPointer_GetStructPointer(p structPointer, f field) structPointer { - return *(*structPointer)(unsafe.Pointer(uintptr(p) + uintptr(f))) +func (p pointer) setInt32Ptr(v int32) { + *(**int32)(p.p) = &v } -// StructPointerSlice the address of a []*struct field in the struct. -func structPointer_StructPointerSlice(p structPointer, f field) *structPointerSlice { - return (*structPointerSlice)(unsafe.Pointer(uintptr(p) + uintptr(f))) +// getInt32Slice loads a []int32 from p. +// The value returned is aliased with the original slice. +// This behavior differs from the implementation in pointer_reflect.go. +func (p pointer) getInt32Slice() []int32 { + return *(*[]int32)(p.p) } -// A structPointerSlice represents a slice of pointers to structs (themselves submessages or groups). -type structPointerSlice []structPointer - -func (v *structPointerSlice) Len() int { return len(*v) } -func (v *structPointerSlice) Index(i int) structPointer { return (*v)[i] } -func (v *structPointerSlice) Append(p structPointer) { *v = append(*v, p) } - -// A word32 is the address of a "pointer to 32-bit value" field. -type word32 **uint32 - -// IsNil reports whether *v is nil. -func word32_IsNil(p word32) bool { - return *p == nil +// setInt32Slice stores a []int32 to p. +// The value set is aliased with the input slice. +// This behavior differs from the implementation in pointer_reflect.go. +func (p pointer) setInt32Slice(v []int32) { + *(*[]int32)(p.p) = v } -// Set sets *v to point at a newly allocated word set to x. -func word32_Set(p word32, o *Buffer, x uint32) { - if len(o.uint32s) == 0 { - o.uint32s = make([]uint32, uint32PoolSize) - } - o.uint32s[0] = x - *p = &o.uint32s[0] - o.uint32s = o.uint32s[1:] +// TODO: Can we get rid of appendInt32Slice and use setInt32Slice instead? +func (p pointer) appendInt32Slice(v int32) { + s := (*[]int32)(p.p) + *s = append(*s, v) } -// Get gets the value pointed at by *v. -func word32_Get(p word32) uint32 { - return **p +func (p pointer) toUint64() *uint64 { + return (*uint64)(p.p) } - -// Word32 returns the address of a *int32, *uint32, *float32, or *enum field in the struct. -func structPointer_Word32(p structPointer, f field) word32 { - return word32((**uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) +func (p pointer) toUint64Ptr() **uint64 { + return (**uint64)(p.p) } - -// A word32Val is the address of a 32-bit value field. -type word32Val *uint32 - -// Set sets *p to x. -func word32Val_Set(p word32Val, x uint32) { - *p = x +func (p pointer) toUint64Slice() *[]uint64 { + return (*[]uint64)(p.p) } - -// Get gets the value pointed at by p. -func word32Val_Get(p word32Val) uint32 { - return *p +func (p pointer) toUint32() *uint32 { + return (*uint32)(p.p) } - -// Word32Val returns the address of a *int32, *uint32, *float32, or *enum field in the struct. -func structPointer_Word32Val(p structPointer, f field) word32Val { - return word32Val((*uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) +func (p pointer) toUint32Ptr() **uint32 { + return (**uint32)(p.p) } - -// A word32Slice is a slice of 32-bit values. -type word32Slice []uint32 - -func (v *word32Slice) Append(x uint32) { *v = append(*v, x) } -func (v *word32Slice) Len() int { return len(*v) } -func (v *word32Slice) Index(i int) uint32 { return (*v)[i] } - -// Word32Slice returns the address of a []int32, []uint32, []float32, or []enum field in the struct. -func structPointer_Word32Slice(p structPointer, f field) *word32Slice { - return (*word32Slice)(unsafe.Pointer(uintptr(p) + uintptr(f))) +func (p pointer) toUint32Slice() *[]uint32 { + return (*[]uint32)(p.p) } - -// word64 is like word32 but for 64-bit values. -type word64 **uint64 - -func word64_Set(p word64, o *Buffer, x uint64) { - if len(o.uint64s) == 0 { - o.uint64s = make([]uint64, uint64PoolSize) - } - o.uint64s[0] = x - *p = &o.uint64s[0] - o.uint64s = o.uint64s[1:] +func (p pointer) toBool() *bool { + return (*bool)(p.p) } - -func word64_IsNil(p word64) bool { - return *p == nil +func (p pointer) toBoolPtr() **bool { + return (**bool)(p.p) } - -func word64_Get(p word64) uint64 { - return **p +func (p pointer) toBoolSlice() *[]bool { + return (*[]bool)(p.p) +} +func (p pointer) toFloat64() *float64 { + return (*float64)(p.p) +} +func (p pointer) toFloat64Ptr() **float64 { + return (**float64)(p.p) +} +func (p pointer) toFloat64Slice() *[]float64 { + return (*[]float64)(p.p) +} +func (p pointer) toFloat32() *float32 { + return (*float32)(p.p) +} +func (p pointer) toFloat32Ptr() **float32 { + return (**float32)(p.p) +} +func (p pointer) toFloat32Slice() *[]float32 { + return (*[]float32)(p.p) +} +func (p pointer) toString() *string { + return (*string)(p.p) +} +func (p pointer) toStringPtr() **string { + return (**string)(p.p) +} +func (p pointer) toStringSlice() *[]string { + return (*[]string)(p.p) +} +func (p pointer) toBytes() *[]byte { + return (*[]byte)(p.p) +} +func (p pointer) toBytesSlice() *[][]byte { + return (*[][]byte)(p.p) +} +func (p pointer) toExtensions() *XXX_InternalExtensions { + return (*XXX_InternalExtensions)(p.p) +} +func (p pointer) toOldExtensions() *map[int32]Extension { + return (*map[int32]Extension)(p.p) } -func structPointer_Word64(p structPointer, f field) word64 { - return word64((**uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) +// getPointerSlice loads []*T from p as a []pointer. +// The value returned is aliased with the original slice. +// This behavior differs from the implementation in pointer_reflect.go. +func (p pointer) getPointerSlice() []pointer { + // Super-tricky - p should point to a []*T where T is a + // message type. We load it as []pointer. + return *(*[]pointer)(p.p) } -// word64Val is like word32Val but for 64-bit values. -type word64Val *uint64 +// setPointerSlice stores []pointer into p as a []*T. +// The value set is aliased with the input slice. +// This behavior differs from the implementation in pointer_reflect.go. +func (p pointer) setPointerSlice(v []pointer) { + // Super-tricky - p should point to a []*T where T is a + // message type. We store it as []pointer. + *(*[]pointer)(p.p) = v +} -func word64Val_Set(p word64Val, o *Buffer, x uint64) { - *p = x +// getPointer loads the pointer at p and returns it. +func (p pointer) getPointer() pointer { + return pointer{p: *(*unsafe.Pointer)(p.p)} } -func word64Val_Get(p word64Val) uint64 { - return *p +// setPointer stores the pointer q at p. +func (p pointer) setPointer(q pointer) { + *(*unsafe.Pointer)(p.p) = q.p } -func structPointer_Word64Val(p structPointer, f field) word64Val { - return word64Val((*uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) +// append q to the slice pointed to by p. +func (p pointer) appendPointer(q pointer) { + s := (*[]unsafe.Pointer)(p.p) + *s = append(*s, q.p) } -// word64Slice is like word32Slice but for 64-bit values. -type word64Slice []uint64 +// getInterfacePointer returns a pointer that points to the +// interface data of the interface pointed by p. +func (p pointer) getInterfacePointer() pointer { + // Super-tricky - read pointer out of data word of interface value. + return pointer{p: (*(*[2]unsafe.Pointer)(p.p))[1]} +} -func (v *word64Slice) Append(x uint64) { *v = append(*v, x) } -func (v *word64Slice) Len() int { return len(*v) } -func (v *word64Slice) Index(i int) uint64 { return (*v)[i] } +// asPointerTo returns a reflect.Value that is a pointer to an +// object of type t stored at p. +func (p pointer) asPointerTo(t reflect.Type) reflect.Value { + return reflect.NewAt(t, p.p) +} -func structPointer_Word64Slice(p structPointer, f field) *word64Slice { - return (*word64Slice)(unsafe.Pointer(uintptr(p) + uintptr(f))) +func atomicLoadUnmarshalInfo(p **unmarshalInfo) *unmarshalInfo { + return (*unmarshalInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p)))) +} +func atomicStoreUnmarshalInfo(p **unmarshalInfo, v *unmarshalInfo) { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v)) +} +func atomicLoadMarshalInfo(p **marshalInfo) *marshalInfo { + return (*marshalInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p)))) +} +func atomicStoreMarshalInfo(p **marshalInfo, v *marshalInfo) { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v)) +} +func atomicLoadMergeInfo(p **mergeInfo) *mergeInfo { + return (*mergeInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p)))) +} +func atomicStoreMergeInfo(p **mergeInfo, v *mergeInfo) { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v)) +} +func atomicLoadDiscardInfo(p **discardInfo) *discardInfo { + return (*discardInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p)))) +} +func atomicStoreDiscardInfo(p **discardInfo, v *discardInfo) { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v)) } diff --git a/vendor/github.com/golang/protobuf/proto/properties.go b/vendor/github.com/golang/protobuf/proto/properties.go index ec2289c0058..f710adab092 100644 --- a/vendor/github.com/golang/protobuf/proto/properties.go +++ b/vendor/github.com/golang/protobuf/proto/properties.go @@ -58,42 +58,6 @@ const ( WireFixed32 = 5 ) -const startSize = 10 // initial slice/string sizes - -// Encoders are defined in encode.go -// An encoder outputs the full representation of a field, including its -// tag and encoder type. -type encoder func(p *Buffer, prop *Properties, base structPointer) error - -// A valueEncoder encodes a single integer in a particular encoding. -type valueEncoder func(o *Buffer, x uint64) error - -// Sizers are defined in encode.go -// A sizer returns the encoded size of a field, including its tag and encoder -// type. -type sizer func(prop *Properties, base structPointer) int - -// A valueSizer returns the encoded size of a single integer in a particular -// encoding. -type valueSizer func(x uint64) int - -// Decoders are defined in decode.go -// A decoder creates a value from its wire representation. -// Unrecognized subelements are saved in unrec. -type decoder func(p *Buffer, prop *Properties, base structPointer) error - -// A valueDecoder decodes a single integer in a particular encoding. -type valueDecoder func(o *Buffer) (x uint64, err error) - -// A oneofMarshaler does the marshaling for all oneof fields in a message. -type oneofMarshaler func(Message, *Buffer) error - -// A oneofUnmarshaler does the unmarshaling for a oneof field in a message. -type oneofUnmarshaler func(Message, int, int, *Buffer) (bool, error) - -// A oneofSizer does the sizing for all oneof fields in a message. -type oneofSizer func(Message) int - // tagMap is an optimization over map[int]int for typical protocol buffer // use-cases. Encoded protocol buffers are often in tag order with small tag // numbers. @@ -140,13 +104,6 @@ type StructProperties struct { decoderTags tagMap // map from proto tag to struct field number decoderOrigNames map[string]int // map from original name to struct field number order []int // list of struct field numbers in tag order - unrecField field // field id of the XXX_unrecognized []byte field - extendable bool // is this an extendable proto - - oneofMarshaler oneofMarshaler - oneofUnmarshaler oneofUnmarshaler - oneofSizer oneofSizer - stype reflect.Type // OneofTypes contains information about the oneof fields in this message. // It is keyed by the original name of a field. @@ -187,36 +144,19 @@ type Properties struct { Default string // default value HasDefault bool // whether an explicit default was provided - def_uint64 uint64 - - enc encoder - valEnc valueEncoder // set for bool and numeric types only - field field - tagcode []byte // encoding of EncodeVarint((Tag<<3)|WireType) - tagbuf [8]byte - stype reflect.Type // set for struct types only - sprop *StructProperties // set for struct types only - isMarshaler bool - isUnmarshaler bool + + stype reflect.Type // set for struct types only + sprop *StructProperties // set for struct types only mtype reflect.Type // set for map types only mkeyprop *Properties // set for map types only mvalprop *Properties // set for map types only - - size sizer - valSize valueSizer // set for bool and numeric types only - - dec decoder - valDec valueDecoder // set for bool and numeric types only - - // If this is a packable field, this will be the decoder for the packed version of the field. - packedDec decoder } // String formats the properties in the protobuf struct field tag style. func (p *Properties) String() string { s := p.Wire - s = "," + s += "," s += strconv.Itoa(p.Tag) if p.Required { s += ",req" @@ -262,29 +202,14 @@ func (p *Properties) Parse(s string) { switch p.Wire { case "varint": p.WireType = WireVarint - p.valEnc = (*Buffer).EncodeVarint - p.valDec = (*Buffer).DecodeVarint - p.valSize = sizeVarint case "fixed32": p.WireType = WireFixed32 - p.valEnc = (*Buffer).EncodeFixed32 - p.valDec = (*Buffer).DecodeFixed32 - p.valSize = sizeFixed32 case "fixed64": p.WireType = WireFixed64 - p.valEnc = (*Buffer).EncodeFixed64 - p.valDec = (*Buffer).DecodeFixed64 - p.valSize = sizeFixed64 case "zigzag32": p.WireType = WireVarint - p.valEnc = (*Buffer).EncodeZigzag32 - p.valDec = (*Buffer).DecodeZigzag32 - p.valSize = sizeZigzag32 case "zigzag64": p.WireType = WireVarint - p.valEnc = (*Buffer).EncodeZigzag64 - p.valDec = (*Buffer).DecodeZigzag64 - p.valSize = sizeZigzag64 case "bytes", "group": p.WireType = WireBytes // no numeric converter for non-numeric types @@ -299,6 +224,7 @@ func (p *Properties) Parse(s string) { return } +outer: for i := 2; i < len(fields); i++ { f := fields[i] switch { @@ -326,229 +252,28 @@ func (p *Properties) Parse(s string) { if i+1 < len(fields) { // Commas aren't escaped, and def is always last. p.Default += "," + strings.Join(fields[i+1:], ",") - break + break outer } } } } -func logNoSliceEnc(t1, t2 reflect.Type) { - fmt.Fprintf(os.Stderr, "proto: no slice oenc for %T = []%T\n", t1, t2) -} - var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem() -// Initialize the fields for encoding and decoding. -func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lockGetProp bool) { - p.enc = nil - p.dec = nil - p.size = nil - +// setFieldProps initializes the field properties for submessages and maps. +func (p *Properties) setFieldProps(typ reflect.Type, f *reflect.StructField, lockGetProp bool) { switch t1 := typ; t1.Kind() { - default: - fmt.Fprintf(os.Stderr, "proto: no coders for %v\n", t1) - - // proto3 scalar types - - case reflect.Bool: - p.enc = (*Buffer).enc_proto3_bool - p.dec = (*Buffer).dec_proto3_bool - p.size = size_proto3_bool - case reflect.Int32: - p.enc = (*Buffer).enc_proto3_int32 - p.dec = (*Buffer).dec_proto3_int32 - p.size = size_proto3_int32 - case reflect.Uint32: - p.enc = (*Buffer).enc_proto3_uint32 - p.dec = (*Buffer).dec_proto3_int32 // can reuse - p.size = size_proto3_uint32 - case reflect.Int64, reflect.Uint64: - p.enc = (*Buffer).enc_proto3_int64 - p.dec = (*Buffer).dec_proto3_int64 - p.size = size_proto3_int64 - case reflect.Float32: - p.enc = (*Buffer).enc_proto3_uint32 // can just treat them as bits - p.dec = (*Buffer).dec_proto3_int32 - p.size = size_proto3_uint32 - case reflect.Float64: - p.enc = (*Buffer).enc_proto3_int64 // can just treat them as bits - p.dec = (*Buffer).dec_proto3_int64 - p.size = size_proto3_int64 - case reflect.String: - p.enc = (*Buffer).enc_proto3_string - p.dec = (*Buffer).dec_proto3_string - p.size = size_proto3_string - case reflect.Ptr: - switch t2 := t1.Elem(); t2.Kind() { - default: - fmt.Fprintf(os.Stderr, "proto: no encoder function for %v -> %v\n", t1, t2) - break - case reflect.Bool: - p.enc = (*Buffer).enc_bool - p.dec = (*Buffer).dec_bool - p.size = size_bool - case reflect.Int32: - p.enc = (*Buffer).enc_int32 - p.dec = (*Buffer).dec_int32 - p.size = size_int32 - case reflect.Uint32: - p.enc = (*Buffer).enc_uint32 - p.dec = (*Buffer).dec_int32 // can reuse - p.size = size_uint32 - case reflect.Int64, reflect.Uint64: - p.enc = (*Buffer).enc_int64 - p.dec = (*Buffer).dec_int64 - p.size = size_int64 - case reflect.Float32: - p.enc = (*Buffer).enc_uint32 // can just treat them as bits - p.dec = (*Buffer).dec_int32 - p.size = size_uint32 - case reflect.Float64: - p.enc = (*Buffer).enc_int64 // can just treat them as bits - p.dec = (*Buffer).dec_int64 - p.size = size_int64 - case reflect.String: - p.enc = (*Buffer).enc_string - p.dec = (*Buffer).dec_string - p.size = size_string - case reflect.Struct: + if t1.Elem().Kind() == reflect.Struct { p.stype = t1.Elem() - p.isMarshaler = isMarshaler(t1) - p.isUnmarshaler = isUnmarshaler(t1) - if p.Wire == "bytes" { - p.enc = (*Buffer).enc_struct_message - p.dec = (*Buffer).dec_struct_message - p.size = size_struct_message - } else { - p.enc = (*Buffer).enc_struct_group - p.dec = (*Buffer).dec_struct_group - p.size = size_struct_group - } } case reflect.Slice: - switch t2 := t1.Elem(); t2.Kind() { - default: - logNoSliceEnc(t1, t2) - break - case reflect.Bool: - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_bool - p.size = size_slice_packed_bool - } else { - p.enc = (*Buffer).enc_slice_bool - p.size = size_slice_bool - } - p.dec = (*Buffer).dec_slice_bool - p.packedDec = (*Buffer).dec_slice_packed_bool - case reflect.Int32: - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_int32 - p.size = size_slice_packed_int32 - } else { - p.enc = (*Buffer).enc_slice_int32 - p.size = size_slice_int32 - } - p.dec = (*Buffer).dec_slice_int32 - p.packedDec = (*Buffer).dec_slice_packed_int32 - case reflect.Uint32: - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_uint32 - p.size = size_slice_packed_uint32 - } else { - p.enc = (*Buffer).enc_slice_uint32 - p.size = size_slice_uint32 - } - p.dec = (*Buffer).dec_slice_int32 - p.packedDec = (*Buffer).dec_slice_packed_int32 - case reflect.Int64, reflect.Uint64: - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_int64 - p.size = size_slice_packed_int64 - } else { - p.enc = (*Buffer).enc_slice_int64 - p.size = size_slice_int64 - } - p.dec = (*Buffer).dec_slice_int64 - p.packedDec = (*Buffer).dec_slice_packed_int64 - case reflect.Uint8: - p.dec = (*Buffer).dec_slice_byte - if p.proto3 { - p.enc = (*Buffer).enc_proto3_slice_byte - p.size = size_proto3_slice_byte - } else { - p.enc = (*Buffer).enc_slice_byte - p.size = size_slice_byte - } - case reflect.Float32, reflect.Float64: - switch t2.Bits() { - case 32: - // can just treat them as bits - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_uint32 - p.size = size_slice_packed_uint32 - } else { - p.enc = (*Buffer).enc_slice_uint32 - p.size = size_slice_uint32 - } - p.dec = (*Buffer).dec_slice_int32 - p.packedDec = (*Buffer).dec_slice_packed_int32 - case 64: - // can just treat them as bits - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_int64 - p.size = size_slice_packed_int64 - } else { - p.enc = (*Buffer).enc_slice_int64 - p.size = size_slice_int64 - } - p.dec = (*Buffer).dec_slice_int64 - p.packedDec = (*Buffer).dec_slice_packed_int64 - default: - logNoSliceEnc(t1, t2) - break - } - case reflect.String: - p.enc = (*Buffer).enc_slice_string - p.dec = (*Buffer).dec_slice_string - p.size = size_slice_string - case reflect.Ptr: - switch t3 := t2.Elem(); t3.Kind() { - default: - fmt.Fprintf(os.Stderr, "proto: no ptr oenc for %T -> %T -> %T\n", t1, t2, t3) - break - case reflect.Struct: - p.stype = t2.Elem() - p.isMarshaler = isMarshaler(t2) - p.isUnmarshaler = isUnmarshaler(t2) - if p.Wire == "bytes" { - p.enc = (*Buffer).enc_slice_struct_message - p.dec = (*Buffer).dec_slice_struct_message - p.size = size_slice_struct_message - } else { - p.enc = (*Buffer).enc_slice_struct_group - p.dec = (*Buffer).dec_slice_struct_group - p.size = size_slice_struct_group - } - } - case reflect.Slice: - switch t2.Elem().Kind() { - default: - fmt.Fprintf(os.Stderr, "proto: no slice elem oenc for %T -> %T -> %T\n", t1, t2, t2.Elem()) - break - case reflect.Uint8: - p.enc = (*Buffer).enc_slice_slice_byte - p.dec = (*Buffer).dec_slice_slice_byte - p.size = size_slice_slice_byte - } + if t2 := t1.Elem(); t2.Kind() == reflect.Ptr && t2.Elem().Kind() == reflect.Struct { + p.stype = t2.Elem() } case reflect.Map: - p.enc = (*Buffer).enc_new_map - p.dec = (*Buffer).dec_new_map - p.size = size_new_map - p.mtype = t1 p.mkeyprop = &Properties{} p.mkeyprop.init(reflect.PtrTo(p.mtype.Key()), "Key", f.Tag.Get("protobuf_key"), nil, lockGetProp) @@ -562,20 +287,6 @@ func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lock p.mvalprop.init(vtype, "Value", f.Tag.Get("protobuf_val"), nil, lockGetProp) } - // precalculate tag code - wire := p.WireType - if p.Packed { - wire = WireBytes - } - x := uint32(p.Tag)<<3 | uint32(wire) - i := 0 - for i = 0; x > 127; i++ { - p.tagbuf[i] = 0x80 | uint8(x&0x7F) - x >>= 7 - } - p.tagbuf[i] = uint8(x) - p.tagcode = p.tagbuf[0 : i+1] - if p.stype != nil { if lockGetProp { p.sprop = GetProperties(p.stype) @@ -586,32 +297,9 @@ func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lock } var ( - marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() - unmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem() + marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() ) -// isMarshaler reports whether type t implements Marshaler. -func isMarshaler(t reflect.Type) bool { - // We're checking for (likely) pointer-receiver methods - // so if t is not a pointer, something is very wrong. - // The calls above only invoke isMarshaler on pointer types. - if t.Kind() != reflect.Ptr { - panic("proto: misuse of isMarshaler") - } - return t.Implements(marshalerType) -} - -// isUnmarshaler reports whether type t implements Unmarshaler. -func isUnmarshaler(t reflect.Type) bool { - // We're checking for (likely) pointer-receiver methods - // so if t is not a pointer, something is very wrong. - // The calls above only invoke isUnmarshaler on pointer types. - if t.Kind() != reflect.Ptr { - panic("proto: misuse of isUnmarshaler") - } - return t.Implements(unmarshalerType) -} - // Init populates the properties from a protocol buffer struct tag. func (p *Properties) Init(typ reflect.Type, name, tag string, f *reflect.StructField) { p.init(typ, name, tag, f, true) @@ -621,14 +309,11 @@ func (p *Properties) init(typ reflect.Type, name, tag string, f *reflect.StructF // "bytes,49,opt,def=hello!" p.Name = name p.OrigName = name - if f != nil { - p.field = toField(f) - } if tag == "" { return } p.Parse(tag) - p.setEncAndDec(typ, f, lockGetProp) + p.setFieldProps(typ, f, lockGetProp) } var ( @@ -678,9 +363,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { propertiesMap[t] = prop // build properties - prop.extendable = reflect.PtrTo(t).Implements(extendableProtoType) || - reflect.PtrTo(t).Implements(extendableProtoV1Type) - prop.unrecField = invalidField prop.Prop = make([]*Properties, t.NumField()) prop.order = make([]int, t.NumField()) @@ -690,17 +372,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { name := f.Name p.init(f.Type, name, f.Tag.Get("protobuf"), &f, false) - if f.Name == "XXX_InternalExtensions" { // special case - p.enc = (*Buffer).enc_exts - p.dec = nil // not needed - p.size = size_exts - } else if f.Name == "XXX_extensions" { // special case - p.enc = (*Buffer).enc_map - p.dec = nil // not needed - p.size = size_map - } else if f.Name == "XXX_unrecognized" { // special case - prop.unrecField = toField(&f) - } oneof := f.Tag.Get("protobuf_oneof") // special case if oneof != "" { // Oneof fields don't use the traditional protobuf tag. @@ -715,9 +386,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { } print("\n") } - if p.enc == nil && !strings.HasPrefix(f.Name, "XXX_") && oneof == "" { - fmt.Fprintln(os.Stderr, "proto: no encoder for", f.Name, f.Type.String(), "[GetProperties]") - } } // Re-order prop.order. @@ -728,8 +396,7 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { } if om, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); ok { var oots []interface{} - prop.oneofMarshaler, prop.oneofUnmarshaler, prop.oneofSizer, oots = om.XXX_OneofFuncs() - prop.stype = t + _, _, _, oots = om.XXX_OneofFuncs() // Interpret oneof metadata. prop.OneofTypes = make(map[string]*OneofProperties) @@ -779,30 +446,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { return prop } -// Return the Properties object for the x[0]'th field of the structure. -func propByIndex(t reflect.Type, x []int) *Properties { - if len(x) != 1 { - fmt.Fprintf(os.Stderr, "proto: field index dimension %d (not 1) for type %s\n", len(x), t) - return nil - } - prop := GetProperties(t) - return prop.Prop[x[0]] -} - -// Get the address and type of a pointer to a struct from an interface. -func getbase(pb Message) (t reflect.Type, b structPointer, err error) { - if pb == nil { - err = ErrNil - return - } - // get the reflect type of the pointer to the struct. - t = reflect.TypeOf(pb) - // get the address of the struct. - value := reflect.ValueOf(pb) - b = toStructPointer(value) - return -} - // A global registry of enum types. // The generated code will register the generated maps by calling RegisterEnum. @@ -826,20 +469,42 @@ func EnumValueMap(enumType string) map[string]int32 { // A registry of all linked message types. // The string is a fully-qualified proto name ("pkg.Message"). var ( - protoTypes = make(map[string]reflect.Type) - revProtoTypes = make(map[reflect.Type]string) + protoTypedNils = make(map[string]Message) // a map from proto names to typed nil pointers + protoMapTypes = make(map[string]reflect.Type) // a map from proto names to map types + revProtoTypes = make(map[reflect.Type]string) ) // RegisterType is called from generated code and maps from the fully qualified // proto name to the type (pointer to struct) of the protocol buffer. func RegisterType(x Message, name string) { - if _, ok := protoTypes[name]; ok { + if _, ok := protoTypedNils[name]; ok { // TODO: Some day, make this a panic. log.Printf("proto: duplicate proto type registered: %s", name) return } t := reflect.TypeOf(x) - protoTypes[name] = t + if v := reflect.ValueOf(x); v.Kind() == reflect.Ptr && v.Pointer() == 0 { + // Generated code always calls RegisterType with nil x. + // This check is just for extra safety. + protoTypedNils[name] = x + } else { + protoTypedNils[name] = reflect.Zero(t).Interface().(Message) + } + revProtoTypes[t] = name +} + +// RegisterMapType is called from generated code and maps from the fully qualified +// proto name to the native map type of the proto map definition. +func RegisterMapType(x interface{}, name string) { + if reflect.TypeOf(x).Kind() != reflect.Map { + panic(fmt.Sprintf("RegisterMapType(%T, %q); want map", x, name)) + } + if _, ok := protoMapTypes[name]; ok { + log.Printf("proto: duplicate proto type registered: %s", name) + return + } + t := reflect.TypeOf(x) + protoMapTypes[name] = t revProtoTypes[t] = name } @@ -855,7 +520,14 @@ func MessageName(x Message) string { } // MessageType returns the message type (pointer to struct) for a named message. -func MessageType(name string) reflect.Type { return protoTypes[name] } +// The type is not guaranteed to implement proto.Message if the name refers to a +// map entry. +func MessageType(name string) reflect.Type { + if t, ok := protoTypedNils[name]; ok { + return reflect.TypeOf(t) + } + return protoMapTypes[name] +} // A registry of all linked proto files. var ( diff --git a/vendor/github.com/golang/protobuf/proto/table_marshal.go b/vendor/github.com/golang/protobuf/proto/table_marshal.go new file mode 100644 index 00000000000..0f212b3029d --- /dev/null +++ b/vendor/github.com/golang/protobuf/proto/table_marshal.go @@ -0,0 +1,2681 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +import ( + "errors" + "fmt" + "math" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "unicode/utf8" +) + +// a sizer takes a pointer to a field and the size of its tag, computes the size of +// the encoded data. +type sizer func(pointer, int) int + +// a marshaler takes a byte slice, a pointer to a field, and its tag (in wire format), +// marshals the field to the end of the slice, returns the slice and error (if any). +type marshaler func(b []byte, ptr pointer, wiretag uint64, deterministic bool) ([]byte, error) + +// marshalInfo is the information used for marshaling a message. +type marshalInfo struct { + typ reflect.Type + fields []*marshalFieldInfo + unrecognized field // offset of XXX_unrecognized + extensions field // offset of XXX_InternalExtensions + v1extensions field // offset of XXX_extensions + sizecache field // offset of XXX_sizecache + initialized int32 // 0 -- only typ is set, 1 -- fully initialized + messageset bool // uses message set wire format + hasmarshaler bool // has custom marshaler + sync.RWMutex // protect extElems map, also for initialization + extElems map[int32]*marshalElemInfo // info of extension elements +} + +// marshalFieldInfo is the information used for marshaling a field of a message. +type marshalFieldInfo struct { + field field + wiretag uint64 // tag in wire format + tagsize int // size of tag in wire format + sizer sizer + marshaler marshaler + isPointer bool + required bool // field is required + name string // name of the field, for error reporting + oneofElems map[reflect.Type]*marshalElemInfo // info of oneof elements +} + +// marshalElemInfo is the information used for marshaling an extension or oneof element. +type marshalElemInfo struct { + wiretag uint64 // tag in wire format + tagsize int // size of tag in wire format + sizer sizer + marshaler marshaler + isptr bool // elem is pointer typed, thus interface of this type is a direct interface (extension only) +} + +var ( + marshalInfoMap = map[reflect.Type]*marshalInfo{} + marshalInfoLock sync.Mutex +) + +// getMarshalInfo returns the information to marshal a given type of message. +// The info it returns may not necessarily initialized. +// t is the type of the message (NOT the pointer to it). +func getMarshalInfo(t reflect.Type) *marshalInfo { + marshalInfoLock.Lock() + u, ok := marshalInfoMap[t] + if !ok { + u = &marshalInfo{typ: t} + marshalInfoMap[t] = u + } + marshalInfoLock.Unlock() + return u +} + +// Size is the entry point from generated code, +// and should be ONLY called by generated code. +// It computes the size of encoded data of msg. +// a is a pointer to a place to store cached marshal info. +func (a *InternalMessageInfo) Size(msg Message) int { + u := getMessageMarshalInfo(msg, a) + ptr := toPointer(&msg) + if ptr.isNil() { + // We get here if msg is a typed nil ((*SomeMessage)(nil)), + // so it satisfies the interface, and msg == nil wouldn't + // catch it. We don't want crash in this case. + return 0 + } + return u.size(ptr) +} + +// Marshal is the entry point from generated code, +// and should be ONLY called by generated code. +// It marshals msg to the end of b. +// a is a pointer to a place to store cached marshal info. +func (a *InternalMessageInfo) Marshal(b []byte, msg Message, deterministic bool) ([]byte, error) { + u := getMessageMarshalInfo(msg, a) + ptr := toPointer(&msg) + if ptr.isNil() { + // We get here if msg is a typed nil ((*SomeMessage)(nil)), + // so it satisfies the interface, and msg == nil wouldn't + // catch it. We don't want crash in this case. + return b, ErrNil + } + return u.marshal(b, ptr, deterministic) +} + +func getMessageMarshalInfo(msg interface{}, a *InternalMessageInfo) *marshalInfo { + // u := a.marshal, but atomically. + // We use an atomic here to ensure memory consistency. + u := atomicLoadMarshalInfo(&a.marshal) + if u == nil { + // Get marshal information from type of message. + t := reflect.ValueOf(msg).Type() + if t.Kind() != reflect.Ptr { + panic(fmt.Sprintf("cannot handle non-pointer message type %v", t)) + } + u = getMarshalInfo(t.Elem()) + // Store it in the cache for later users. + // a.marshal = u, but atomically. + atomicStoreMarshalInfo(&a.marshal, u) + } + return u +} + +// size is the main function to compute the size of the encoded data of a message. +// ptr is the pointer to the message. +func (u *marshalInfo) size(ptr pointer) int { + if atomic.LoadInt32(&u.initialized) == 0 { + u.computeMarshalInfo() + } + + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + if u.hasmarshaler { + m := ptr.asPointerTo(u.typ).Interface().(Marshaler) + b, _ := m.Marshal() + return len(b) + } + + n := 0 + for _, f := range u.fields { + if f.isPointer && ptr.offset(f.field).getPointer().isNil() { + // nil pointer always marshals to nothing + continue + } + n += f.sizer(ptr.offset(f.field), f.tagsize) + } + if u.extensions.IsValid() { + e := ptr.offset(u.extensions).toExtensions() + if u.messageset { + n += u.sizeMessageSet(e) + } else { + n += u.sizeExtensions(e) + } + } + if u.v1extensions.IsValid() { + m := *ptr.offset(u.v1extensions).toOldExtensions() + n += u.sizeV1Extensions(m) + } + if u.unrecognized.IsValid() { + s := *ptr.offset(u.unrecognized).toBytes() + n += len(s) + } + // cache the result for use in marshal + if u.sizecache.IsValid() { + atomic.StoreInt32(ptr.offset(u.sizecache).toInt32(), int32(n)) + } + return n +} + +// cachedsize gets the size from cache. If there is no cache (i.e. message is not generated), +// fall back to compute the size. +func (u *marshalInfo) cachedsize(ptr pointer) int { + if u.sizecache.IsValid() { + return int(atomic.LoadInt32(ptr.offset(u.sizecache).toInt32())) + } + return u.size(ptr) +} + +// marshal is the main function to marshal a message. It takes a byte slice and appends +// the encoded data to the end of the slice, returns the slice and error (if any). +// ptr is the pointer to the message. +// If deterministic is true, map is marshaled in deterministic order. +func (u *marshalInfo) marshal(b []byte, ptr pointer, deterministic bool) ([]byte, error) { + if atomic.LoadInt32(&u.initialized) == 0 { + u.computeMarshalInfo() + } + + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + if u.hasmarshaler { + m := ptr.asPointerTo(u.typ).Interface().(Marshaler) + b1, err := m.Marshal() + b = append(b, b1...) + return b, err + } + + var err, errreq error + // The old marshaler encodes extensions at beginning. + if u.extensions.IsValid() { + e := ptr.offset(u.extensions).toExtensions() + if u.messageset { + b, err = u.appendMessageSet(b, e, deterministic) + } else { + b, err = u.appendExtensions(b, e, deterministic) + } + if err != nil { + return b, err + } + } + if u.v1extensions.IsValid() { + m := *ptr.offset(u.v1extensions).toOldExtensions() + b, err = u.appendV1Extensions(b, m, deterministic) + if err != nil { + return b, err + } + } + for _, f := range u.fields { + if f.required && errreq == nil { + if ptr.offset(f.field).getPointer().isNil() { + // Required field is not set. + // We record the error but keep going, to give a complete marshaling. + errreq = &RequiredNotSetError{f.name} + continue + } + } + if f.isPointer && ptr.offset(f.field).getPointer().isNil() { + // nil pointer always marshals to nothing + continue + } + b, err = f.marshaler(b, ptr.offset(f.field), f.wiretag, deterministic) + if err != nil { + if err1, ok := err.(*RequiredNotSetError); ok { + // Required field in submessage is not set. + // We record the error but keep going, to give a complete marshaling. + if errreq == nil { + errreq = &RequiredNotSetError{f.name + "." + err1.field} + } + continue + } + if err == errRepeatedHasNil { + err = errors.New("proto: repeated field " + f.name + " has nil element") + } + return b, err + } + } + if u.unrecognized.IsValid() { + s := *ptr.offset(u.unrecognized).toBytes() + b = append(b, s...) + } + return b, errreq +} + +// computeMarshalInfo initializes the marshal info. +func (u *marshalInfo) computeMarshalInfo() { + u.Lock() + defer u.Unlock() + if u.initialized != 0 { // non-atomic read is ok as it is protected by the lock + return + } + + t := u.typ + u.unrecognized = invalidField + u.extensions = invalidField + u.v1extensions = invalidField + u.sizecache = invalidField + + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + if reflect.PtrTo(t).Implements(marshalerType) { + u.hasmarshaler = true + atomic.StoreInt32(&u.initialized, 1) + return + } + + // get oneof implementers + var oneofImplementers []interface{} + if m, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); ok { + _, _, _, oneofImplementers = m.XXX_OneofFuncs() + } + + n := t.NumField() + + // deal with XXX fields first + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if !strings.HasPrefix(f.Name, "XXX_") { + continue + } + switch f.Name { + case "XXX_sizecache": + u.sizecache = toField(&f) + case "XXX_unrecognized": + u.unrecognized = toField(&f) + case "XXX_InternalExtensions": + u.extensions = toField(&f) + u.messageset = f.Tag.Get("protobuf_messageset") == "1" + case "XXX_extensions": + u.v1extensions = toField(&f) + case "XXX_NoUnkeyedLiteral": + // nothing to do + default: + panic("unknown XXX field: " + f.Name) + } + n-- + } + + // normal fields + fields := make([]marshalFieldInfo, n) // batch allocation + u.fields = make([]*marshalFieldInfo, 0, n) + for i, j := 0, 0; i < t.NumField(); i++ { + f := t.Field(i) + + if strings.HasPrefix(f.Name, "XXX_") { + continue + } + field := &fields[j] + j++ + field.name = f.Name + u.fields = append(u.fields, field) + if f.Tag.Get("protobuf_oneof") != "" { + field.computeOneofFieldInfo(&f, oneofImplementers) + continue + } + if f.Tag.Get("protobuf") == "" { + // field has no tag (not in generated message), ignore it + u.fields = u.fields[:len(u.fields)-1] + j-- + continue + } + field.computeMarshalFieldInfo(&f) + } + + // fields are marshaled in tag order on the wire. + sort.Sort(byTag(u.fields)) + + atomic.StoreInt32(&u.initialized, 1) +} + +// helper for sorting fields by tag +type byTag []*marshalFieldInfo + +func (a byTag) Len() int { return len(a) } +func (a byTag) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byTag) Less(i, j int) bool { return a[i].wiretag < a[j].wiretag } + +// getExtElemInfo returns the information to marshal an extension element. +// The info it returns is initialized. +func (u *marshalInfo) getExtElemInfo(desc *ExtensionDesc) *marshalElemInfo { + // get from cache first + u.RLock() + e, ok := u.extElems[desc.Field] + u.RUnlock() + if ok { + return e + } + + t := reflect.TypeOf(desc.ExtensionType) // pointer or slice to basic type or struct + tags := strings.Split(desc.Tag, ",") + tag, err := strconv.Atoi(tags[1]) + if err != nil { + panic("tag is not an integer") + } + wt := wiretype(tags[0]) + sizer, marshaler := typeMarshaler(t, tags, false, false) + e = &marshalElemInfo{ + wiretag: uint64(tag)<<3 | wt, + tagsize: SizeVarint(uint64(tag) << 3), + sizer: sizer, + marshaler: marshaler, + isptr: t.Kind() == reflect.Ptr, + } + + // update cache + u.Lock() + if u.extElems == nil { + u.extElems = make(map[int32]*marshalElemInfo) + } + u.extElems[desc.Field] = e + u.Unlock() + return e +} + +// computeMarshalFieldInfo fills up the information to marshal a field. +func (fi *marshalFieldInfo) computeMarshalFieldInfo(f *reflect.StructField) { + // parse protobuf tag of the field. + // tag has format of "bytes,49,opt,name=foo,def=hello!" + tags := strings.Split(f.Tag.Get("protobuf"), ",") + if tags[0] == "" { + return + } + tag, err := strconv.Atoi(tags[1]) + if err != nil { + panic("tag is not an integer") + } + wt := wiretype(tags[0]) + if tags[2] == "req" { + fi.required = true + } + fi.setTag(f, tag, wt) + fi.setMarshaler(f, tags) +} + +func (fi *marshalFieldInfo) computeOneofFieldInfo(f *reflect.StructField, oneofImplementers []interface{}) { + fi.field = toField(f) + fi.wiretag = 1<<31 - 1 // Use a large tag number, make oneofs sorted at the end. This tag will not appear on the wire. + fi.isPointer = true + fi.sizer, fi.marshaler = makeOneOfMarshaler(fi, f) + fi.oneofElems = make(map[reflect.Type]*marshalElemInfo) + + ityp := f.Type // interface type + for _, o := range oneofImplementers { + t := reflect.TypeOf(o) + if !t.Implements(ityp) { + continue + } + sf := t.Elem().Field(0) // oneof implementer is a struct with a single field + tags := strings.Split(sf.Tag.Get("protobuf"), ",") + tag, err := strconv.Atoi(tags[1]) + if err != nil { + panic("tag is not an integer") + } + wt := wiretype(tags[0]) + sizer, marshaler := typeMarshaler(sf.Type, tags, false, true) // oneof should not omit any zero value + fi.oneofElems[t.Elem()] = &marshalElemInfo{ + wiretag: uint64(tag)<<3 | wt, + tagsize: SizeVarint(uint64(tag) << 3), + sizer: sizer, + marshaler: marshaler, + } + } +} + +type oneofMessage interface { + XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{}) +} + +// wiretype returns the wire encoding of the type. +func wiretype(encoding string) uint64 { + switch encoding { + case "fixed32": + return WireFixed32 + case "fixed64": + return WireFixed64 + case "varint", "zigzag32", "zigzag64": + return WireVarint + case "bytes": + return WireBytes + case "group": + return WireStartGroup + } + panic("unknown wire type " + encoding) +} + +// setTag fills up the tag (in wire format) and its size in the info of a field. +func (fi *marshalFieldInfo) setTag(f *reflect.StructField, tag int, wt uint64) { + fi.field = toField(f) + fi.wiretag = uint64(tag)<<3 | wt + fi.tagsize = SizeVarint(uint64(tag) << 3) +} + +// setMarshaler fills up the sizer and marshaler in the info of a field. +func (fi *marshalFieldInfo) setMarshaler(f *reflect.StructField, tags []string) { + switch f.Type.Kind() { + case reflect.Map: + // map field + fi.isPointer = true + fi.sizer, fi.marshaler = makeMapMarshaler(f) + return + case reflect.Ptr, reflect.Slice: + fi.isPointer = true + } + fi.sizer, fi.marshaler = typeMarshaler(f.Type, tags, true, false) +} + +// typeMarshaler returns the sizer and marshaler of a given field. +// t is the type of the field. +// tags is the generated "protobuf" tag of the field. +// If nozero is true, zero value is not marshaled to the wire. +// If oneof is true, it is a oneof field. +func typeMarshaler(t reflect.Type, tags []string, nozero, oneof bool) (sizer, marshaler) { + encoding := tags[0] + + pointer := false + slice := false + if t.Kind() == reflect.Slice && t.Elem().Kind() != reflect.Uint8 { + slice = true + t = t.Elem() + } + if t.Kind() == reflect.Ptr { + pointer = true + t = t.Elem() + } + + packed := false + proto3 := false + for i := 2; i < len(tags); i++ { + if tags[i] == "packed" { + packed = true + } + if tags[i] == "proto3" { + proto3 = true + } + } + + switch t.Kind() { + case reflect.Bool: + if pointer { + return sizeBoolPtr, appendBoolPtr + } + if slice { + if packed { + return sizeBoolPackedSlice, appendBoolPackedSlice + } + return sizeBoolSlice, appendBoolSlice + } + if nozero { + return sizeBoolValueNoZero, appendBoolValueNoZero + } + return sizeBoolValue, appendBoolValue + case reflect.Uint32: + switch encoding { + case "fixed32": + if pointer { + return sizeFixed32Ptr, appendFixed32Ptr + } + if slice { + if packed { + return sizeFixed32PackedSlice, appendFixed32PackedSlice + } + return sizeFixed32Slice, appendFixed32Slice + } + if nozero { + return sizeFixed32ValueNoZero, appendFixed32ValueNoZero + } + return sizeFixed32Value, appendFixed32Value + case "varint": + if pointer { + return sizeVarint32Ptr, appendVarint32Ptr + } + if slice { + if packed { + return sizeVarint32PackedSlice, appendVarint32PackedSlice + } + return sizeVarint32Slice, appendVarint32Slice + } + if nozero { + return sizeVarint32ValueNoZero, appendVarint32ValueNoZero + } + return sizeVarint32Value, appendVarint32Value + } + case reflect.Int32: + switch encoding { + case "fixed32": + if pointer { + return sizeFixedS32Ptr, appendFixedS32Ptr + } + if slice { + if packed { + return sizeFixedS32PackedSlice, appendFixedS32PackedSlice + } + return sizeFixedS32Slice, appendFixedS32Slice + } + if nozero { + return sizeFixedS32ValueNoZero, appendFixedS32ValueNoZero + } + return sizeFixedS32Value, appendFixedS32Value + case "varint": + if pointer { + return sizeVarintS32Ptr, appendVarintS32Ptr + } + if slice { + if packed { + return sizeVarintS32PackedSlice, appendVarintS32PackedSlice + } + return sizeVarintS32Slice, appendVarintS32Slice + } + if nozero { + return sizeVarintS32ValueNoZero, appendVarintS32ValueNoZero + } + return sizeVarintS32Value, appendVarintS32Value + case "zigzag32": + if pointer { + return sizeZigzag32Ptr, appendZigzag32Ptr + } + if slice { + if packed { + return sizeZigzag32PackedSlice, appendZigzag32PackedSlice + } + return sizeZigzag32Slice, appendZigzag32Slice + } + if nozero { + return sizeZigzag32ValueNoZero, appendZigzag32ValueNoZero + } + return sizeZigzag32Value, appendZigzag32Value + } + case reflect.Uint64: + switch encoding { + case "fixed64": + if pointer { + return sizeFixed64Ptr, appendFixed64Ptr + } + if slice { + if packed { + return sizeFixed64PackedSlice, appendFixed64PackedSlice + } + return sizeFixed64Slice, appendFixed64Slice + } + if nozero { + return sizeFixed64ValueNoZero, appendFixed64ValueNoZero + } + return sizeFixed64Value, appendFixed64Value + case "varint": + if pointer { + return sizeVarint64Ptr, appendVarint64Ptr + } + if slice { + if packed { + return sizeVarint64PackedSlice, appendVarint64PackedSlice + } + return sizeVarint64Slice, appendVarint64Slice + } + if nozero { + return sizeVarint64ValueNoZero, appendVarint64ValueNoZero + } + return sizeVarint64Value, appendVarint64Value + } + case reflect.Int64: + switch encoding { + case "fixed64": + if pointer { + return sizeFixedS64Ptr, appendFixedS64Ptr + } + if slice { + if packed { + return sizeFixedS64PackedSlice, appendFixedS64PackedSlice + } + return sizeFixedS64Slice, appendFixedS64Slice + } + if nozero { + return sizeFixedS64ValueNoZero, appendFixedS64ValueNoZero + } + return sizeFixedS64Value, appendFixedS64Value + case "varint": + if pointer { + return sizeVarintS64Ptr, appendVarintS64Ptr + } + if slice { + if packed { + return sizeVarintS64PackedSlice, appendVarintS64PackedSlice + } + return sizeVarintS64Slice, appendVarintS64Slice + } + if nozero { + return sizeVarintS64ValueNoZero, appendVarintS64ValueNoZero + } + return sizeVarintS64Value, appendVarintS64Value + case "zigzag64": + if pointer { + return sizeZigzag64Ptr, appendZigzag64Ptr + } + if slice { + if packed { + return sizeZigzag64PackedSlice, appendZigzag64PackedSlice + } + return sizeZigzag64Slice, appendZigzag64Slice + } + if nozero { + return sizeZigzag64ValueNoZero, appendZigzag64ValueNoZero + } + return sizeZigzag64Value, appendZigzag64Value + } + case reflect.Float32: + if pointer { + return sizeFloat32Ptr, appendFloat32Ptr + } + if slice { + if packed { + return sizeFloat32PackedSlice, appendFloat32PackedSlice + } + return sizeFloat32Slice, appendFloat32Slice + } + if nozero { + return sizeFloat32ValueNoZero, appendFloat32ValueNoZero + } + return sizeFloat32Value, appendFloat32Value + case reflect.Float64: + if pointer { + return sizeFloat64Ptr, appendFloat64Ptr + } + if slice { + if packed { + return sizeFloat64PackedSlice, appendFloat64PackedSlice + } + return sizeFloat64Slice, appendFloat64Slice + } + if nozero { + return sizeFloat64ValueNoZero, appendFloat64ValueNoZero + } + return sizeFloat64Value, appendFloat64Value + case reflect.String: + if pointer { + return sizeStringPtr, appendStringPtr + } + if slice { + return sizeStringSlice, appendStringSlice + } + if nozero { + return sizeStringValueNoZero, appendStringValueNoZero + } + return sizeStringValue, appendStringValue + case reflect.Slice: + if slice { + return sizeBytesSlice, appendBytesSlice + } + if oneof { + // Oneof bytes field may also have "proto3" tag. + // We want to marshal it as a oneof field. Do this + // check before the proto3 check. + return sizeBytesOneof, appendBytesOneof + } + if proto3 { + return sizeBytes3, appendBytes3 + } + return sizeBytes, appendBytes + case reflect.Struct: + switch encoding { + case "group": + if slice { + return makeGroupSliceMarshaler(getMarshalInfo(t)) + } + return makeGroupMarshaler(getMarshalInfo(t)) + case "bytes": + if slice { + return makeMessageSliceMarshaler(getMarshalInfo(t)) + } + return makeMessageMarshaler(getMarshalInfo(t)) + } + } + panic(fmt.Sprintf("unknown or mismatched type: type: %v, wire type: %v", t, encoding)) +} + +// Below are functions to size/marshal a specific type of a field. +// They are stored in the field's info, and called by function pointers. +// They have type sizer or marshaler. + +func sizeFixed32Value(_ pointer, tagsize int) int { + return 4 + tagsize +} +func sizeFixed32ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toUint32() + if v == 0 { + return 0 + } + return 4 + tagsize +} +func sizeFixed32Ptr(ptr pointer, tagsize int) int { + p := *ptr.toUint32Ptr() + if p == nil { + return 0 + } + return 4 + tagsize +} +func sizeFixed32Slice(ptr pointer, tagsize int) int { + s := *ptr.toUint32Slice() + return (4 + tagsize) * len(s) +} +func sizeFixed32PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toUint32Slice() + if len(s) == 0 { + return 0 + } + return 4*len(s) + SizeVarint(uint64(4*len(s))) + tagsize +} +func sizeFixedS32Value(_ pointer, tagsize int) int { + return 4 + tagsize +} +func sizeFixedS32ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt32() + if v == 0 { + return 0 + } + return 4 + tagsize +} +func sizeFixedS32Ptr(ptr pointer, tagsize int) int { + p := ptr.getInt32Ptr() + if p == nil { + return 0 + } + return 4 + tagsize +} +func sizeFixedS32Slice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + return (4 + tagsize) * len(s) +} +func sizeFixedS32PackedSlice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + if len(s) == 0 { + return 0 + } + return 4*len(s) + SizeVarint(uint64(4*len(s))) + tagsize +} +func sizeFloat32Value(_ pointer, tagsize int) int { + return 4 + tagsize +} +func sizeFloat32ValueNoZero(ptr pointer, tagsize int) int { + v := math.Float32bits(*ptr.toFloat32()) + if v == 0 { + return 0 + } + return 4 + tagsize +} +func sizeFloat32Ptr(ptr pointer, tagsize int) int { + p := *ptr.toFloat32Ptr() + if p == nil { + return 0 + } + return 4 + tagsize +} +func sizeFloat32Slice(ptr pointer, tagsize int) int { + s := *ptr.toFloat32Slice() + return (4 + tagsize) * len(s) +} +func sizeFloat32PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toFloat32Slice() + if len(s) == 0 { + return 0 + } + return 4*len(s) + SizeVarint(uint64(4*len(s))) + tagsize +} +func sizeFixed64Value(_ pointer, tagsize int) int { + return 8 + tagsize +} +func sizeFixed64ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toUint64() + if v == 0 { + return 0 + } + return 8 + tagsize +} +func sizeFixed64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toUint64Ptr() + if p == nil { + return 0 + } + return 8 + tagsize +} +func sizeFixed64Slice(ptr pointer, tagsize int) int { + s := *ptr.toUint64Slice() + return (8 + tagsize) * len(s) +} +func sizeFixed64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toUint64Slice() + if len(s) == 0 { + return 0 + } + return 8*len(s) + SizeVarint(uint64(8*len(s))) + tagsize +} +func sizeFixedS64Value(_ pointer, tagsize int) int { + return 8 + tagsize +} +func sizeFixedS64ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt64() + if v == 0 { + return 0 + } + return 8 + tagsize +} +func sizeFixedS64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toInt64Ptr() + if p == nil { + return 0 + } + return 8 + tagsize +} +func sizeFixedS64Slice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + return (8 + tagsize) * len(s) +} +func sizeFixedS64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return 0 + } + return 8*len(s) + SizeVarint(uint64(8*len(s))) + tagsize +} +func sizeFloat64Value(_ pointer, tagsize int) int { + return 8 + tagsize +} +func sizeFloat64ValueNoZero(ptr pointer, tagsize int) int { + v := math.Float64bits(*ptr.toFloat64()) + if v == 0 { + return 0 + } + return 8 + tagsize +} +func sizeFloat64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toFloat64Ptr() + if p == nil { + return 0 + } + return 8 + tagsize +} +func sizeFloat64Slice(ptr pointer, tagsize int) int { + s := *ptr.toFloat64Slice() + return (8 + tagsize) * len(s) +} +func sizeFloat64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toFloat64Slice() + if len(s) == 0 { + return 0 + } + return 8*len(s) + SizeVarint(uint64(8*len(s))) + tagsize +} +func sizeVarint32Value(ptr pointer, tagsize int) int { + v := *ptr.toUint32() + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarint32ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toUint32() + if v == 0 { + return 0 + } + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarint32Ptr(ptr pointer, tagsize int) int { + p := *ptr.toUint32Ptr() + if p == nil { + return 0 + } + return SizeVarint(uint64(*p)) + tagsize +} +func sizeVarint32Slice(ptr pointer, tagsize int) int { + s := *ptr.toUint32Slice() + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + tagsize + } + return n +} +func sizeVarint32PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toUint32Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeVarintS32Value(ptr pointer, tagsize int) int { + v := *ptr.toInt32() + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarintS32ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt32() + if v == 0 { + return 0 + } + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarintS32Ptr(ptr pointer, tagsize int) int { + p := ptr.getInt32Ptr() + if p == nil { + return 0 + } + return SizeVarint(uint64(*p)) + tagsize +} +func sizeVarintS32Slice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + tagsize + } + return n +} +func sizeVarintS32PackedSlice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeVarint64Value(ptr pointer, tagsize int) int { + v := *ptr.toUint64() + return SizeVarint(v) + tagsize +} +func sizeVarint64ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toUint64() + if v == 0 { + return 0 + } + return SizeVarint(v) + tagsize +} +func sizeVarint64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toUint64Ptr() + if p == nil { + return 0 + } + return SizeVarint(*p) + tagsize +} +func sizeVarint64Slice(ptr pointer, tagsize int) int { + s := *ptr.toUint64Slice() + n := 0 + for _, v := range s { + n += SizeVarint(v) + tagsize + } + return n +} +func sizeVarint64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toUint64Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(v) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeVarintS64Value(ptr pointer, tagsize int) int { + v := *ptr.toInt64() + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarintS64ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt64() + if v == 0 { + return 0 + } + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarintS64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toInt64Ptr() + if p == nil { + return 0 + } + return SizeVarint(uint64(*p)) + tagsize +} +func sizeVarintS64Slice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + tagsize + } + return n +} +func sizeVarintS64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeZigzag32Value(ptr pointer, tagsize int) int { + v := *ptr.toInt32() + return SizeVarint(uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + tagsize +} +func sizeZigzag32ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt32() + if v == 0 { + return 0 + } + return SizeVarint(uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + tagsize +} +func sizeZigzag32Ptr(ptr pointer, tagsize int) int { + p := ptr.getInt32Ptr() + if p == nil { + return 0 + } + v := *p + return SizeVarint(uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + tagsize +} +func sizeZigzag32Slice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + n := 0 + for _, v := range s { + n += SizeVarint(uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + tagsize + } + return n +} +func sizeZigzag32PackedSlice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(uint64((uint32(v) << 1) ^ uint32((int32(v) >> 31)))) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeZigzag64Value(ptr pointer, tagsize int) int { + v := *ptr.toInt64() + return SizeVarint(uint64(v<<1)^uint64((int64(v)>>63))) + tagsize +} +func sizeZigzag64ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt64() + if v == 0 { + return 0 + } + return SizeVarint(uint64(v<<1)^uint64((int64(v)>>63))) + tagsize +} +func sizeZigzag64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toInt64Ptr() + if p == nil { + return 0 + } + v := *p + return SizeVarint(uint64(v<<1)^uint64((int64(v)>>63))) + tagsize +} +func sizeZigzag64Slice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v<<1)^uint64((int64(v)>>63))) + tagsize + } + return n +} +func sizeZigzag64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v<<1) ^ uint64((int64(v) >> 63))) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeBoolValue(_ pointer, tagsize int) int { + return 1 + tagsize +} +func sizeBoolValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toBool() + if !v { + return 0 + } + return 1 + tagsize +} +func sizeBoolPtr(ptr pointer, tagsize int) int { + p := *ptr.toBoolPtr() + if p == nil { + return 0 + } + return 1 + tagsize +} +func sizeBoolSlice(ptr pointer, tagsize int) int { + s := *ptr.toBoolSlice() + return (1 + tagsize) * len(s) +} +func sizeBoolPackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toBoolSlice() + if len(s) == 0 { + return 0 + } + return len(s) + SizeVarint(uint64(len(s))) + tagsize +} +func sizeStringValue(ptr pointer, tagsize int) int { + v := *ptr.toString() + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeStringValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toString() + if v == "" { + return 0 + } + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeStringPtr(ptr pointer, tagsize int) int { + p := *ptr.toStringPtr() + if p == nil { + return 0 + } + v := *p + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeStringSlice(ptr pointer, tagsize int) int { + s := *ptr.toStringSlice() + n := 0 + for _, v := range s { + n += len(v) + SizeVarint(uint64(len(v))) + tagsize + } + return n +} +func sizeBytes(ptr pointer, tagsize int) int { + v := *ptr.toBytes() + if v == nil { + return 0 + } + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeBytes3(ptr pointer, tagsize int) int { + v := *ptr.toBytes() + if len(v) == 0 { + return 0 + } + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeBytesOneof(ptr pointer, tagsize int) int { + v := *ptr.toBytes() + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeBytesSlice(ptr pointer, tagsize int) int { + s := *ptr.toBytesSlice() + n := 0 + for _, v := range s { + n += len(v) + SizeVarint(uint64(len(v))) + tagsize + } + return n +} + +// appendFixed32 appends an encoded fixed32 to b. +func appendFixed32(b []byte, v uint32) []byte { + b = append(b, + byte(v), + byte(v>>8), + byte(v>>16), + byte(v>>24)) + return b +} + +// appendFixed64 appends an encoded fixed64 to b. +func appendFixed64(b []byte, v uint64) []byte { + b = append(b, + byte(v), + byte(v>>8), + byte(v>>16), + byte(v>>24), + byte(v>>32), + byte(v>>40), + byte(v>>48), + byte(v>>56)) + return b +} + +// appendVarint appends an encoded varint to b. +func appendVarint(b []byte, v uint64) []byte { + // TODO: make 1-byte (maybe 2-byte) case inline-able, once we + // have non-leaf inliner. + switch { + case v < 1<<7: + b = append(b, byte(v)) + case v < 1<<14: + b = append(b, + byte(v&0x7f|0x80), + byte(v>>7)) + case v < 1<<21: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte(v>>14)) + case v < 1<<28: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte(v>>21)) + case v < 1<<35: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte(v>>28)) + case v < 1<<42: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte(v>>35)) + case v < 1<<49: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte(v>>42)) + case v < 1<<56: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte((v>>42)&0x7f|0x80), + byte(v>>49)) + case v < 1<<63: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte((v>>42)&0x7f|0x80), + byte((v>>49)&0x7f|0x80), + byte(v>>56)) + default: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte((v>>42)&0x7f|0x80), + byte((v>>49)&0x7f|0x80), + byte((v>>56)&0x7f|0x80), + 1) + } + return b +} + +func appendFixed32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint32() + b = appendVarint(b, wiretag) + b = appendFixed32(b, v) + return b, nil +} +func appendFixed32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint32() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, v) + return b, nil +} +func appendFixed32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toUint32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, *p) + return b, nil +} +func appendFixed32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed32(b, v) + } + return b, nil +} +func appendFixed32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(4*len(s))) + for _, v := range s { + b = appendFixed32(b, v) + } + return b, nil +} +func appendFixedS32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + b = appendVarint(b, wiretag) + b = appendFixed32(b, uint32(v)) + return b, nil +} +func appendFixedS32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, uint32(v)) + return b, nil +} +func appendFixedS32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := ptr.getInt32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, uint32(*p)) + return b, nil +} +func appendFixedS32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed32(b, uint32(v)) + } + return b, nil +} +func appendFixedS32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(4*len(s))) + for _, v := range s { + b = appendFixed32(b, uint32(v)) + } + return b, nil +} +func appendFloat32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := math.Float32bits(*ptr.toFloat32()) + b = appendVarint(b, wiretag) + b = appendFixed32(b, v) + return b, nil +} +func appendFloat32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := math.Float32bits(*ptr.toFloat32()) + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, v) + return b, nil +} +func appendFloat32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toFloat32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, math.Float32bits(*p)) + return b, nil +} +func appendFloat32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toFloat32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed32(b, math.Float32bits(v)) + } + return b, nil +} +func appendFloat32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toFloat32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(4*len(s))) + for _, v := range s { + b = appendFixed32(b, math.Float32bits(v)) + } + return b, nil +} +func appendFixed64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint64() + b = appendVarint(b, wiretag) + b = appendFixed64(b, v) + return b, nil +} +func appendFixed64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint64() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, v) + return b, nil +} +func appendFixed64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toUint64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, *p) + return b, nil +} +func appendFixed64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed64(b, v) + } + return b, nil +} +func appendFixed64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(8*len(s))) + for _, v := range s { + b = appendFixed64(b, v) + } + return b, nil +} +func appendFixedS64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + b = appendVarint(b, wiretag) + b = appendFixed64(b, uint64(v)) + return b, nil +} +func appendFixedS64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, uint64(v)) + return b, nil +} +func appendFixedS64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toInt64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, uint64(*p)) + return b, nil +} +func appendFixedS64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed64(b, uint64(v)) + } + return b, nil +} +func appendFixedS64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(8*len(s))) + for _, v := range s { + b = appendFixed64(b, uint64(v)) + } + return b, nil +} +func appendFloat64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := math.Float64bits(*ptr.toFloat64()) + b = appendVarint(b, wiretag) + b = appendFixed64(b, v) + return b, nil +} +func appendFloat64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := math.Float64bits(*ptr.toFloat64()) + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, v) + return b, nil +} +func appendFloat64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toFloat64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, math.Float64bits(*p)) + return b, nil +} +func appendFloat64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toFloat64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed64(b, math.Float64bits(v)) + } + return b, nil +} +func appendFloat64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toFloat64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(8*len(s))) + for _, v := range s { + b = appendFixed64(b, math.Float64bits(v)) + } + return b, nil +} +func appendVarint32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint32() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarint32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint32() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarint32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toUint32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(*p)) + return b, nil +} +func appendVarint32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendVarint32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendVarintS32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarintS32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarintS32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := ptr.getInt32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(*p)) + return b, nil +} +func appendVarintS32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendVarintS32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendVarint64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint64() + b = appendVarint(b, wiretag) + b = appendVarint(b, v) + return b, nil +} +func appendVarint64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint64() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, v) + return b, nil +} +func appendVarint64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toUint64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, *p) + return b, nil +} +func appendVarint64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, v) + } + return b, nil +} +func appendVarint64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(v) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, v) + } + return b, nil +} +func appendVarintS64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarintS64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarintS64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toInt64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(*p)) + return b, nil +} +func appendVarintS64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendVarintS64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendZigzag32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + return b, nil +} +func appendZigzag32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + return b, nil +} +func appendZigzag32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := ptr.getInt32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + v := *p + b = appendVarint(b, uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + return b, nil +} +func appendZigzag32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + } + return b, nil +} +func appendZigzag32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(uint64((uint32(v) << 1) ^ uint32((int32(v) >> 31)))) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + } + return b, nil +} +func appendZigzag64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v<<1)^uint64((int64(v)>>63))) + return b, nil +} +func appendZigzag64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v<<1)^uint64((int64(v)>>63))) + return b, nil +} +func appendZigzag64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toInt64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + v := *p + b = appendVarint(b, uint64(v<<1)^uint64((int64(v)>>63))) + return b, nil +} +func appendZigzag64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v<<1)^uint64((int64(v)>>63))) + } + return b, nil +} +func appendZigzag64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v<<1) ^ uint64((int64(v) >> 63))) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, uint64(v<<1)^uint64((int64(v)>>63))) + } + return b, nil +} +func appendBoolValue(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toBool() + b = appendVarint(b, wiretag) + if v { + b = append(b, 1) + } else { + b = append(b, 0) + } + return b, nil +} +func appendBoolValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toBool() + if !v { + return b, nil + } + b = appendVarint(b, wiretag) + b = append(b, 1) + return b, nil +} + +func appendBoolPtr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toBoolPtr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + if *p { + b = append(b, 1) + } else { + b = append(b, 0) + } + return b, nil +} +func appendBoolSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toBoolSlice() + for _, v := range s { + b = appendVarint(b, wiretag) + if v { + b = append(b, 1) + } else { + b = append(b, 0) + } + } + return b, nil +} +func appendBoolPackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toBoolSlice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(len(s))) + for _, v := range s { + if v { + b = append(b, 1) + } else { + b = append(b, 0) + } + } + return b, nil +} +func appendStringValue(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toString() + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendStringValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toString() + if v == "" { + return b, nil + } + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendStringPtr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toStringPtr() + if p == nil { + return b, nil + } + v := *p + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendStringSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toStringSlice() + for _, v := range s { + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + } + return b, nil +} +func appendBytes(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toBytes() + if v == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendBytes3(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toBytes() + if len(v) == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendBytesOneof(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toBytes() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendBytesSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toBytesSlice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + } + return b, nil +} + +// makeGroupMarshaler returns the sizer and marshaler for a group. +// u is the marshal info of the underlying message. +func makeGroupMarshaler(u *marshalInfo) (sizer, marshaler) { + return func(ptr pointer, tagsize int) int { + p := ptr.getPointer() + if p.isNil() { + return 0 + } + return u.size(p) + 2*tagsize + }, + func(b []byte, ptr pointer, wiretag uint64, deterministic bool) ([]byte, error) { + p := ptr.getPointer() + if p.isNil() { + return b, nil + } + var err error + b = appendVarint(b, wiretag) // start group + b, err = u.marshal(b, p, deterministic) + b = appendVarint(b, wiretag+(WireEndGroup-WireStartGroup)) // end group + return b, err + } +} + +// makeGroupSliceMarshaler returns the sizer and marshaler for a group slice. +// u is the marshal info of the underlying message. +func makeGroupSliceMarshaler(u *marshalInfo) (sizer, marshaler) { + return func(ptr pointer, tagsize int) int { + s := ptr.getPointerSlice() + n := 0 + for _, v := range s { + if v.isNil() { + continue + } + n += u.size(v) + 2*tagsize + } + return n + }, + func(b []byte, ptr pointer, wiretag uint64, deterministic bool) ([]byte, error) { + s := ptr.getPointerSlice() + var err, errreq error + for _, v := range s { + if v.isNil() { + return b, errRepeatedHasNil + } + b = appendVarint(b, wiretag) // start group + b, err = u.marshal(b, v, deterministic) + b = appendVarint(b, wiretag+(WireEndGroup-WireStartGroup)) // end group + if err != nil { + if _, ok := err.(*RequiredNotSetError); ok { + // Required field in submessage is not set. + // We record the error but keep going, to give a complete marshaling. + if errreq == nil { + errreq = err + } + continue + } + if err == ErrNil { + err = errRepeatedHasNil + } + return b, err + } + } + return b, errreq + } +} + +// makeMessageMarshaler returns the sizer and marshaler for a message field. +// u is the marshal info of the message. +func makeMessageMarshaler(u *marshalInfo) (sizer, marshaler) { + return func(ptr pointer, tagsize int) int { + p := ptr.getPointer() + if p.isNil() { + return 0 + } + siz := u.size(p) + return siz + SizeVarint(uint64(siz)) + tagsize + }, + func(b []byte, ptr pointer, wiretag uint64, deterministic bool) ([]byte, error) { + p := ptr.getPointer() + if p.isNil() { + return b, nil + } + b = appendVarint(b, wiretag) + siz := u.cachedsize(p) + b = appendVarint(b, uint64(siz)) + return u.marshal(b, p, deterministic) + } +} + +// makeMessageSliceMarshaler returns the sizer and marshaler for a message slice. +// u is the marshal info of the message. +func makeMessageSliceMarshaler(u *marshalInfo) (sizer, marshaler) { + return func(ptr pointer, tagsize int) int { + s := ptr.getPointerSlice() + n := 0 + for _, v := range s { + if v.isNil() { + continue + } + siz := u.size(v) + n += siz + SizeVarint(uint64(siz)) + tagsize + } + return n + }, + func(b []byte, ptr pointer, wiretag uint64, deterministic bool) ([]byte, error) { + s := ptr.getPointerSlice() + var err, errreq error + for _, v := range s { + if v.isNil() { + return b, errRepeatedHasNil + } + b = appendVarint(b, wiretag) + siz := u.cachedsize(v) + b = appendVarint(b, uint64(siz)) + b, err = u.marshal(b, v, deterministic) + + if err != nil { + if _, ok := err.(*RequiredNotSetError); ok { + // Required field in submessage is not set. + // We record the error but keep going, to give a complete marshaling. + if errreq == nil { + errreq = err + } + continue + } + if err == ErrNil { + err = errRepeatedHasNil + } + return b, err + } + } + return b, errreq + } +} + +// makeMapMarshaler returns the sizer and marshaler for a map field. +// f is the pointer to the reflect data structure of the field. +func makeMapMarshaler(f *reflect.StructField) (sizer, marshaler) { + // figure out key and value type + t := f.Type + keyType := t.Key() + valType := t.Elem() + keyTags := strings.Split(f.Tag.Get("protobuf_key"), ",") + valTags := strings.Split(f.Tag.Get("protobuf_val"), ",") + keySizer, keyMarshaler := typeMarshaler(keyType, keyTags, false, false) // don't omit zero value in map + valSizer, valMarshaler := typeMarshaler(valType, valTags, false, false) // don't omit zero value in map + keyWireTag := 1<<3 | wiretype(keyTags[0]) + valWireTag := 2<<3 | wiretype(valTags[0]) + + // We create an interface to get the addresses of the map key and value. + // If value is pointer-typed, the interface is a direct interface, the + // idata itself is the value. Otherwise, the idata is the pointer to the + // value. + // Key cannot be pointer-typed. + valIsPtr := valType.Kind() == reflect.Ptr + return func(ptr pointer, tagsize int) int { + m := ptr.asPointerTo(t).Elem() // the map + n := 0 + for _, k := range m.MapKeys() { + ki := k.Interface() + vi := m.MapIndex(k).Interface() + kaddr := toAddrPointer(&ki, false) // pointer to key + vaddr := toAddrPointer(&vi, valIsPtr) // pointer to value + siz := keySizer(kaddr, 1) + valSizer(vaddr, 1) // tag of key = 1 (size=1), tag of val = 2 (size=1) + n += siz + SizeVarint(uint64(siz)) + tagsize + } + return n + }, + func(b []byte, ptr pointer, tag uint64, deterministic bool) ([]byte, error) { + m := ptr.asPointerTo(t).Elem() // the map + var err error + keys := m.MapKeys() + if len(keys) > 1 && deterministic { + sort.Sort(mapKeys(keys)) + } + for _, k := range keys { + ki := k.Interface() + vi := m.MapIndex(k).Interface() + kaddr := toAddrPointer(&ki, false) // pointer to key + vaddr := toAddrPointer(&vi, valIsPtr) // pointer to value + b = appendVarint(b, tag) + siz := keySizer(kaddr, 1) + valSizer(vaddr, 1) // tag of key = 1 (size=1), tag of val = 2 (size=1) + b = appendVarint(b, uint64(siz)) + b, err = keyMarshaler(b, kaddr, keyWireTag, deterministic) + if err != nil { + return b, err + } + b, err = valMarshaler(b, vaddr, valWireTag, deterministic) + if err != nil && err != ErrNil { // allow nil value in map + return b, err + } + } + return b, nil + } +} + +// makeOneOfMarshaler returns the sizer and marshaler for a oneof field. +// fi is the marshal info of the field. +// f is the pointer to the reflect data structure of the field. +func makeOneOfMarshaler(fi *marshalFieldInfo, f *reflect.StructField) (sizer, marshaler) { + // Oneof field is an interface. We need to get the actual data type on the fly. + t := f.Type + return func(ptr pointer, _ int) int { + p := ptr.getInterfacePointer() + if p.isNil() { + return 0 + } + v := ptr.asPointerTo(t).Elem().Elem().Elem() // *interface -> interface -> *struct -> struct + telem := v.Type() + e := fi.oneofElems[telem] + return e.sizer(p, e.tagsize) + }, + func(b []byte, ptr pointer, _ uint64, deterministic bool) ([]byte, error) { + p := ptr.getInterfacePointer() + if p.isNil() { + return b, nil + } + v := ptr.asPointerTo(t).Elem().Elem().Elem() // *interface -> interface -> *struct -> struct + telem := v.Type() + if telem.Field(0).Type.Kind() == reflect.Ptr && p.getPointer().isNil() { + return b, errOneofHasNil + } + e := fi.oneofElems[telem] + return e.marshaler(b, p, e.wiretag, deterministic) + } +} + +// sizeExtensions computes the size of encoded data for a XXX_InternalExtensions field. +func (u *marshalInfo) sizeExtensions(ext *XXX_InternalExtensions) int { + m, mu := ext.extensionsRead() + if m == nil { + return 0 + } + mu.Lock() + + n := 0 + for _, e := range m { + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + n += len(e.enc) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + n += ei.sizer(p, ei.tagsize) + } + mu.Unlock() + return n +} + +// appendExtensions marshals a XXX_InternalExtensions field to the end of byte slice b. +func (u *marshalInfo) appendExtensions(b []byte, ext *XXX_InternalExtensions, deterministic bool) ([]byte, error) { + m, mu := ext.extensionsRead() + if m == nil { + return b, nil + } + mu.Lock() + defer mu.Unlock() + + var err error + + // Fast-path for common cases: zero or one extensions. + // Don't bother sorting the keys. + if len(m) <= 1 { + for _, e := range m { + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + b = append(b, e.enc...) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + b, err = ei.marshaler(b, p, ei.wiretag, deterministic) + if err != nil { + return b, err + } + } + return b, nil + } + + // Sort the keys to provide a deterministic encoding. + // Not sure this is required, but the old code does it. + keys := make([]int, 0, len(m)) + for k := range m { + keys = append(keys, int(k)) + } + sort.Ints(keys) + + for _, k := range keys { + e := m[int32(k)] + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + b = append(b, e.enc...) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + b, err = ei.marshaler(b, p, ei.wiretag, deterministic) + if err != nil { + return b, err + } + } + return b, nil +} + +// message set format is: +// message MessageSet { +// repeated group Item = 1 { +// required int32 type_id = 2; +// required string message = 3; +// }; +// } + +// sizeMessageSet computes the size of encoded data for a XXX_InternalExtensions field +// in message set format (above). +func (u *marshalInfo) sizeMessageSet(ext *XXX_InternalExtensions) int { + m, mu := ext.extensionsRead() + if m == nil { + return 0 + } + mu.Lock() + + n := 0 + for id, e := range m { + n += 2 // start group, end group. tag = 1 (size=1) + n += SizeVarint(uint64(id)) + 1 // type_id, tag = 2 (size=1) + + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + msgWithLen := skipVarint(e.enc) // skip old tag, but leave the length varint + siz := len(msgWithLen) + n += siz + 1 // message, tag = 3 (size=1) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + n += ei.sizer(p, 1) // message, tag = 3 (size=1) + } + mu.Unlock() + return n +} + +// appendMessageSet marshals a XXX_InternalExtensions field in message set format (above) +// to the end of byte slice b. +func (u *marshalInfo) appendMessageSet(b []byte, ext *XXX_InternalExtensions, deterministic bool) ([]byte, error) { + m, mu := ext.extensionsRead() + if m == nil { + return b, nil + } + mu.Lock() + defer mu.Unlock() + + var err error + + // Fast-path for common cases: zero or one extensions. + // Don't bother sorting the keys. + if len(m) <= 1 { + for id, e := range m { + b = append(b, 1<<3|WireStartGroup) + b = append(b, 2<<3|WireVarint) + b = appendVarint(b, uint64(id)) + + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + msgWithLen := skipVarint(e.enc) // skip old tag, but leave the length varint + b = append(b, 3<<3|WireBytes) + b = append(b, msgWithLen...) + b = append(b, 1<<3|WireEndGroup) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + b, err = ei.marshaler(b, p, 3<<3|WireBytes, deterministic) + if err != nil { + return b, err + } + b = append(b, 1<<3|WireEndGroup) + } + return b, nil + } + + // Sort the keys to provide a deterministic encoding. + keys := make([]int, 0, len(m)) + for k := range m { + keys = append(keys, int(k)) + } + sort.Ints(keys) + + for _, id := range keys { + e := m[int32(id)] + b = append(b, 1<<3|WireStartGroup) + b = append(b, 2<<3|WireVarint) + b = appendVarint(b, uint64(id)) + + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + msgWithLen := skipVarint(e.enc) // skip old tag, but leave the length varint + b = append(b, 3<<3|WireBytes) + b = append(b, msgWithLen...) + b = append(b, 1<<3|WireEndGroup) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + b, err = ei.marshaler(b, p, 3<<3|WireBytes, deterministic) + b = append(b, 1<<3|WireEndGroup) + if err != nil { + return b, err + } + } + return b, nil +} + +// sizeV1Extensions computes the size of encoded data for a V1-API extension field. +func (u *marshalInfo) sizeV1Extensions(m map[int32]Extension) int { + if m == nil { + return 0 + } + + n := 0 + for _, e := range m { + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + n += len(e.enc) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + n += ei.sizer(p, ei.tagsize) + } + return n +} + +// appendV1Extensions marshals a V1-API extension field to the end of byte slice b. +func (u *marshalInfo) appendV1Extensions(b []byte, m map[int32]Extension, deterministic bool) ([]byte, error) { + if m == nil { + return b, nil + } + + // Sort the keys to provide a deterministic encoding. + keys := make([]int, 0, len(m)) + for k := range m { + keys = append(keys, int(k)) + } + sort.Ints(keys) + + var err error + for _, k := range keys { + e := m[int32(k)] + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + b = append(b, e.enc...) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + b, err = ei.marshaler(b, p, ei.wiretag, deterministic) + if err != nil { + return b, err + } + } + return b, nil +} + +// newMarshaler is the interface representing objects that can marshal themselves. +// +// This exists to support protoc-gen-go generated messages. +// The proto package will stop type-asserting to this interface in the future. +// +// DO NOT DEPEND ON THIS. +type newMarshaler interface { + XXX_Size() int + XXX_Marshal(b []byte, deterministic bool) ([]byte, error) +} + +// Size returns the encoded size of a protocol buffer message. +// This is the main entry point. +func Size(pb Message) int { + if m, ok := pb.(newMarshaler); ok { + return m.XXX_Size() + } + if m, ok := pb.(Marshaler); ok { + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + b, _ := m.Marshal() + return len(b) + } + // in case somehow we didn't generate the wrapper + if pb == nil { + return 0 + } + var info InternalMessageInfo + return info.Size(pb) +} + +// Marshal takes a protocol buffer message +// and encodes it into the wire format, returning the data. +// This is the main entry point. +func Marshal(pb Message) ([]byte, error) { + if m, ok := pb.(newMarshaler); ok { + siz := m.XXX_Size() + b := make([]byte, 0, siz) + return m.XXX_Marshal(b, false) + } + if m, ok := pb.(Marshaler); ok { + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + return m.Marshal() + } + // in case somehow we didn't generate the wrapper + if pb == nil { + return nil, ErrNil + } + var info InternalMessageInfo + siz := info.Size(pb) + b := make([]byte, 0, siz) + return info.Marshal(b, pb, false) +} + +// Marshal takes a protocol buffer message +// and encodes it into the wire format, writing the result to the +// Buffer. +// This is an alternative entry point. It is not necessary to use +// a Buffer for most applications. +func (p *Buffer) Marshal(pb Message) error { + var err error + if m, ok := pb.(newMarshaler); ok { + siz := m.XXX_Size() + p.grow(siz) // make sure buf has enough capacity + p.buf, err = m.XXX_Marshal(p.buf, p.deterministic) + return err + } + if m, ok := pb.(Marshaler); ok { + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + b, err := m.Marshal() + p.buf = append(p.buf, b...) + return err + } + // in case somehow we didn't generate the wrapper + if pb == nil { + return ErrNil + } + var info InternalMessageInfo + siz := info.Size(pb) + p.grow(siz) // make sure buf has enough capacity + p.buf, err = info.Marshal(p.buf, pb, p.deterministic) + return err +} + +// grow grows the buffer's capacity, if necessary, to guarantee space for +// another n bytes. After grow(n), at least n bytes can be written to the +// buffer without another allocation. +func (p *Buffer) grow(n int) { + need := len(p.buf) + n + if need <= cap(p.buf) { + return + } + newCap := len(p.buf) * 2 + if newCap < need { + newCap = need + } + p.buf = append(make([]byte, 0, newCap), p.buf...) +} diff --git a/vendor/github.com/golang/protobuf/proto/table_merge.go b/vendor/github.com/golang/protobuf/proto/table_merge.go new file mode 100644 index 00000000000..5525def6a5d --- /dev/null +++ b/vendor/github.com/golang/protobuf/proto/table_merge.go @@ -0,0 +1,654 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +import ( + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" +) + +// Merge merges the src message into dst. +// This assumes that dst and src of the same type and are non-nil. +func (a *InternalMessageInfo) Merge(dst, src Message) { + mi := atomicLoadMergeInfo(&a.merge) + if mi == nil { + mi = getMergeInfo(reflect.TypeOf(dst).Elem()) + atomicStoreMergeInfo(&a.merge, mi) + } + mi.merge(toPointer(&dst), toPointer(&src)) +} + +type mergeInfo struct { + typ reflect.Type + + initialized int32 // 0: only typ is valid, 1: everything is valid + lock sync.Mutex + + fields []mergeFieldInfo + unrecognized field // Offset of XXX_unrecognized +} + +type mergeFieldInfo struct { + field field // Offset of field, guaranteed to be valid + + // isPointer reports whether the value in the field is a pointer. + // This is true for the following situations: + // * Pointer to struct + // * Pointer to basic type (proto2 only) + // * Slice (first value in slice header is a pointer) + // * String (first value in string header is a pointer) + isPointer bool + + // basicWidth reports the width of the field assuming that it is directly + // embedded in the struct (as is the case for basic types in proto3). + // The possible values are: + // 0: invalid + // 1: bool + // 4: int32, uint32, float32 + // 8: int64, uint64, float64 + basicWidth int + + // Where dst and src are pointers to the types being merged. + merge func(dst, src pointer) +} + +var ( + mergeInfoMap = map[reflect.Type]*mergeInfo{} + mergeInfoLock sync.Mutex +) + +func getMergeInfo(t reflect.Type) *mergeInfo { + mergeInfoLock.Lock() + defer mergeInfoLock.Unlock() + mi := mergeInfoMap[t] + if mi == nil { + mi = &mergeInfo{typ: t} + mergeInfoMap[t] = mi + } + return mi +} + +// merge merges src into dst assuming they are both of type *mi.typ. +func (mi *mergeInfo) merge(dst, src pointer) { + if dst.isNil() { + panic("proto: nil destination") + } + if src.isNil() { + return // Nothing to do. + } + + if atomic.LoadInt32(&mi.initialized) == 0 { + mi.computeMergeInfo() + } + + for _, fi := range mi.fields { + sfp := src.offset(fi.field) + + // As an optimization, we can avoid the merge function call cost + // if we know for sure that the source will have no effect + // by checking if it is the zero value. + if unsafeAllowed { + if fi.isPointer && sfp.getPointer().isNil() { // Could be slice or string + continue + } + if fi.basicWidth > 0 { + switch { + case fi.basicWidth == 1 && !*sfp.toBool(): + continue + case fi.basicWidth == 4 && *sfp.toUint32() == 0: + continue + case fi.basicWidth == 8 && *sfp.toUint64() == 0: + continue + } + } + } + + dfp := dst.offset(fi.field) + fi.merge(dfp, sfp) + } + + // TODO: Make this faster? + out := dst.asPointerTo(mi.typ).Elem() + in := src.asPointerTo(mi.typ).Elem() + if emIn, err := extendable(in.Addr().Interface()); err == nil { + emOut, _ := extendable(out.Addr().Interface()) + mIn, muIn := emIn.extensionsRead() + if mIn != nil { + mOut := emOut.extensionsWrite() + muIn.Lock() + mergeExtension(mOut, mIn) + muIn.Unlock() + } + } + + if mi.unrecognized.IsValid() { + if b := *src.offset(mi.unrecognized).toBytes(); len(b) > 0 { + *dst.offset(mi.unrecognized).toBytes() = append([]byte(nil), b...) + } + } +} + +func (mi *mergeInfo) computeMergeInfo() { + mi.lock.Lock() + defer mi.lock.Unlock() + if mi.initialized != 0 { + return + } + t := mi.typ + n := t.NumField() + + props := GetProperties(t) + for i := 0; i < n; i++ { + f := t.Field(i) + if strings.HasPrefix(f.Name, "XXX_") { + continue + } + + mfi := mergeFieldInfo{field: toField(&f)} + tf := f.Type + + // As an optimization, we can avoid the merge function call cost + // if we know for sure that the source will have no effect + // by checking if it is the zero value. + if unsafeAllowed { + switch tf.Kind() { + case reflect.Ptr, reflect.Slice, reflect.String: + // As a special case, we assume slices and strings are pointers + // since we know that the first field in the SliceSlice or + // StringHeader is a data pointer. + mfi.isPointer = true + case reflect.Bool: + mfi.basicWidth = 1 + case reflect.Int32, reflect.Uint32, reflect.Float32: + mfi.basicWidth = 4 + case reflect.Int64, reflect.Uint64, reflect.Float64: + mfi.basicWidth = 8 + } + } + + // Unwrap tf to get at its most basic type. + var isPointer, isSlice bool + if tf.Kind() == reflect.Slice && tf.Elem().Kind() != reflect.Uint8 { + isSlice = true + tf = tf.Elem() + } + if tf.Kind() == reflect.Ptr { + isPointer = true + tf = tf.Elem() + } + if isPointer && isSlice && tf.Kind() != reflect.Struct { + panic("both pointer and slice for basic type in " + tf.Name()) + } + + switch tf.Kind() { + case reflect.Int32: + switch { + case isSlice: // E.g., []int32 + mfi.merge = func(dst, src pointer) { + // NOTE: toInt32Slice is not defined (see pointer_reflect.go). + /* + sfsp := src.toInt32Slice() + if *sfsp != nil { + dfsp := dst.toInt32Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []int64{} + } + } + */ + sfs := src.getInt32Slice() + if sfs != nil { + dfs := dst.getInt32Slice() + dfs = append(dfs, sfs...) + if dfs == nil { + dfs = []int32{} + } + dst.setInt32Slice(dfs) + } + } + case isPointer: // E.g., *int32 + mfi.merge = func(dst, src pointer) { + // NOTE: toInt32Ptr is not defined (see pointer_reflect.go). + /* + sfpp := src.toInt32Ptr() + if *sfpp != nil { + dfpp := dst.toInt32Ptr() + if *dfpp == nil { + *dfpp = Int32(**sfpp) + } else { + **dfpp = **sfpp + } + } + */ + sfp := src.getInt32Ptr() + if sfp != nil { + dfp := dst.getInt32Ptr() + if dfp == nil { + dst.setInt32Ptr(*sfp) + } else { + *dfp = *sfp + } + } + } + default: // E.g., int32 + mfi.merge = func(dst, src pointer) { + if v := *src.toInt32(); v != 0 { + *dst.toInt32() = v + } + } + } + case reflect.Int64: + switch { + case isSlice: // E.g., []int64 + mfi.merge = func(dst, src pointer) { + sfsp := src.toInt64Slice() + if *sfsp != nil { + dfsp := dst.toInt64Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []int64{} + } + } + } + case isPointer: // E.g., *int64 + mfi.merge = func(dst, src pointer) { + sfpp := src.toInt64Ptr() + if *sfpp != nil { + dfpp := dst.toInt64Ptr() + if *dfpp == nil { + *dfpp = Int64(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., int64 + mfi.merge = func(dst, src pointer) { + if v := *src.toInt64(); v != 0 { + *dst.toInt64() = v + } + } + } + case reflect.Uint32: + switch { + case isSlice: // E.g., []uint32 + mfi.merge = func(dst, src pointer) { + sfsp := src.toUint32Slice() + if *sfsp != nil { + dfsp := dst.toUint32Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []uint32{} + } + } + } + case isPointer: // E.g., *uint32 + mfi.merge = func(dst, src pointer) { + sfpp := src.toUint32Ptr() + if *sfpp != nil { + dfpp := dst.toUint32Ptr() + if *dfpp == nil { + *dfpp = Uint32(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., uint32 + mfi.merge = func(dst, src pointer) { + if v := *src.toUint32(); v != 0 { + *dst.toUint32() = v + } + } + } + case reflect.Uint64: + switch { + case isSlice: // E.g., []uint64 + mfi.merge = func(dst, src pointer) { + sfsp := src.toUint64Slice() + if *sfsp != nil { + dfsp := dst.toUint64Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []uint64{} + } + } + } + case isPointer: // E.g., *uint64 + mfi.merge = func(dst, src pointer) { + sfpp := src.toUint64Ptr() + if *sfpp != nil { + dfpp := dst.toUint64Ptr() + if *dfpp == nil { + *dfpp = Uint64(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., uint64 + mfi.merge = func(dst, src pointer) { + if v := *src.toUint64(); v != 0 { + *dst.toUint64() = v + } + } + } + case reflect.Float32: + switch { + case isSlice: // E.g., []float32 + mfi.merge = func(dst, src pointer) { + sfsp := src.toFloat32Slice() + if *sfsp != nil { + dfsp := dst.toFloat32Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []float32{} + } + } + } + case isPointer: // E.g., *float32 + mfi.merge = func(dst, src pointer) { + sfpp := src.toFloat32Ptr() + if *sfpp != nil { + dfpp := dst.toFloat32Ptr() + if *dfpp == nil { + *dfpp = Float32(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., float32 + mfi.merge = func(dst, src pointer) { + if v := *src.toFloat32(); v != 0 { + *dst.toFloat32() = v + } + } + } + case reflect.Float64: + switch { + case isSlice: // E.g., []float64 + mfi.merge = func(dst, src pointer) { + sfsp := src.toFloat64Slice() + if *sfsp != nil { + dfsp := dst.toFloat64Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []float64{} + } + } + } + case isPointer: // E.g., *float64 + mfi.merge = func(dst, src pointer) { + sfpp := src.toFloat64Ptr() + if *sfpp != nil { + dfpp := dst.toFloat64Ptr() + if *dfpp == nil { + *dfpp = Float64(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., float64 + mfi.merge = func(dst, src pointer) { + if v := *src.toFloat64(); v != 0 { + *dst.toFloat64() = v + } + } + } + case reflect.Bool: + switch { + case isSlice: // E.g., []bool + mfi.merge = func(dst, src pointer) { + sfsp := src.toBoolSlice() + if *sfsp != nil { + dfsp := dst.toBoolSlice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []bool{} + } + } + } + case isPointer: // E.g., *bool + mfi.merge = func(dst, src pointer) { + sfpp := src.toBoolPtr() + if *sfpp != nil { + dfpp := dst.toBoolPtr() + if *dfpp == nil { + *dfpp = Bool(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., bool + mfi.merge = func(dst, src pointer) { + if v := *src.toBool(); v { + *dst.toBool() = v + } + } + } + case reflect.String: + switch { + case isSlice: // E.g., []string + mfi.merge = func(dst, src pointer) { + sfsp := src.toStringSlice() + if *sfsp != nil { + dfsp := dst.toStringSlice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []string{} + } + } + } + case isPointer: // E.g., *string + mfi.merge = func(dst, src pointer) { + sfpp := src.toStringPtr() + if *sfpp != nil { + dfpp := dst.toStringPtr() + if *dfpp == nil { + *dfpp = String(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., string + mfi.merge = func(dst, src pointer) { + if v := *src.toString(); v != "" { + *dst.toString() = v + } + } + } + case reflect.Slice: + isProto3 := props.Prop[i].proto3 + switch { + case isPointer: + panic("bad pointer in byte slice case in " + tf.Name()) + case tf.Elem().Kind() != reflect.Uint8: + panic("bad element kind in byte slice case in " + tf.Name()) + case isSlice: // E.g., [][]byte + mfi.merge = func(dst, src pointer) { + sbsp := src.toBytesSlice() + if *sbsp != nil { + dbsp := dst.toBytesSlice() + for _, sb := range *sbsp { + if sb == nil { + *dbsp = append(*dbsp, nil) + } else { + *dbsp = append(*dbsp, append([]byte{}, sb...)) + } + } + if *dbsp == nil { + *dbsp = [][]byte{} + } + } + } + default: // E.g., []byte + mfi.merge = func(dst, src pointer) { + sbp := src.toBytes() + if *sbp != nil { + dbp := dst.toBytes() + if !isProto3 || len(*sbp) > 0 { + *dbp = append([]byte{}, *sbp...) + } + } + } + } + case reflect.Struct: + switch { + case !isPointer: + panic(fmt.Sprintf("message field %s without pointer", tf)) + case isSlice: // E.g., []*pb.T + mi := getMergeInfo(tf) + mfi.merge = func(dst, src pointer) { + sps := src.getPointerSlice() + if sps != nil { + dps := dst.getPointerSlice() + for _, sp := range sps { + var dp pointer + if !sp.isNil() { + dp = valToPointer(reflect.New(tf)) + mi.merge(dp, sp) + } + dps = append(dps, dp) + } + if dps == nil { + dps = []pointer{} + } + dst.setPointerSlice(dps) + } + } + default: // E.g., *pb.T + mi := getMergeInfo(tf) + mfi.merge = func(dst, src pointer) { + sp := src.getPointer() + if !sp.isNil() { + dp := dst.getPointer() + if dp.isNil() { + dp = valToPointer(reflect.New(tf)) + dst.setPointer(dp) + } + mi.merge(dp, sp) + } + } + } + case reflect.Map: + switch { + case isPointer || isSlice: + panic("bad pointer or slice in map case in " + tf.Name()) + default: // E.g., map[K]V + mfi.merge = func(dst, src pointer) { + sm := src.asPointerTo(tf).Elem() + if sm.Len() == 0 { + return + } + dm := dst.asPointerTo(tf).Elem() + if dm.IsNil() { + dm.Set(reflect.MakeMap(tf)) + } + + switch tf.Elem().Kind() { + case reflect.Ptr: // Proto struct (e.g., *T) + for _, key := range sm.MapKeys() { + val := sm.MapIndex(key) + val = reflect.ValueOf(Clone(val.Interface().(Message))) + dm.SetMapIndex(key, val) + } + case reflect.Slice: // E.g. Bytes type (e.g., []byte) + for _, key := range sm.MapKeys() { + val := sm.MapIndex(key) + val = reflect.ValueOf(append([]byte{}, val.Bytes()...)) + dm.SetMapIndex(key, val) + } + default: // Basic type (e.g., string) + for _, key := range sm.MapKeys() { + val := sm.MapIndex(key) + dm.SetMapIndex(key, val) + } + } + } + } + case reflect.Interface: + // Must be oneof field. + switch { + case isPointer || isSlice: + panic("bad pointer or slice in interface case in " + tf.Name()) + default: // E.g., interface{} + // TODO: Make this faster? + mfi.merge = func(dst, src pointer) { + su := src.asPointerTo(tf).Elem() + if !su.IsNil() { + du := dst.asPointerTo(tf).Elem() + typ := su.Elem().Type() + if du.IsNil() || du.Elem().Type() != typ { + du.Set(reflect.New(typ.Elem())) // Initialize interface if empty + } + sv := su.Elem().Elem().Field(0) + if sv.Kind() == reflect.Ptr && sv.IsNil() { + return + } + dv := du.Elem().Elem().Field(0) + if dv.Kind() == reflect.Ptr && dv.IsNil() { + dv.Set(reflect.New(sv.Type().Elem())) // Initialize proto message if empty + } + switch sv.Type().Kind() { + case reflect.Ptr: // Proto struct (e.g., *T) + Merge(dv.Interface().(Message), sv.Interface().(Message)) + case reflect.Slice: // E.g. Bytes type (e.g., []byte) + dv.Set(reflect.ValueOf(append([]byte{}, sv.Bytes()...))) + default: // Basic type (e.g., string) + dv.Set(sv) + } + } + } + } + default: + panic(fmt.Sprintf("merger not found for type:%s", tf)) + } + mi.fields = append(mi.fields, mfi) + } + + mi.unrecognized = invalidField + if f, ok := t.FieldByName("XXX_unrecognized"); ok { + if f.Type != reflect.TypeOf([]byte{}) { + panic("expected XXX_unrecognized to be of type []byte") + } + mi.unrecognized = toField(&f) + } + + atomic.StoreInt32(&mi.initialized, 1) +} diff --git a/vendor/github.com/golang/protobuf/proto/table_unmarshal.go b/vendor/github.com/golang/protobuf/proto/table_unmarshal.go new file mode 100644 index 00000000000..55f0340a3fd --- /dev/null +++ b/vendor/github.com/golang/protobuf/proto/table_unmarshal.go @@ -0,0 +1,1967 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +import ( + "errors" + "fmt" + "io" + "math" + "reflect" + "strconv" + "strings" + "sync" + "sync/atomic" + "unicode/utf8" +) + +// Unmarshal is the entry point from the generated .pb.go files. +// This function is not intended to be used by non-generated code. +// This function is not subject to any compatibility guarantee. +// msg contains a pointer to a protocol buffer struct. +// b is the data to be unmarshaled into the protocol buffer. +// a is a pointer to a place to store cached unmarshal information. +func (a *InternalMessageInfo) Unmarshal(msg Message, b []byte) error { + // Load the unmarshal information for this message type. + // The atomic load ensures memory consistency. + u := atomicLoadUnmarshalInfo(&a.unmarshal) + if u == nil { + // Slow path: find unmarshal info for msg, update a with it. + u = getUnmarshalInfo(reflect.TypeOf(msg).Elem()) + atomicStoreUnmarshalInfo(&a.unmarshal, u) + } + // Then do the unmarshaling. + err := u.unmarshal(toPointer(&msg), b) + return err +} + +type unmarshalInfo struct { + typ reflect.Type // type of the protobuf struct + + // 0 = only typ field is initialized + // 1 = completely initialized + initialized int32 + lock sync.Mutex // prevents double initialization + dense []unmarshalFieldInfo // fields indexed by tag # + sparse map[uint64]unmarshalFieldInfo // fields indexed by tag # + reqFields []string // names of required fields + reqMask uint64 // 1< 0 { + // Read tag and wire type. + // Special case 1 and 2 byte varints. + var x uint64 + if b[0] < 128 { + x = uint64(b[0]) + b = b[1:] + } else if len(b) >= 2 && b[1] < 128 { + x = uint64(b[0]&0x7f) + uint64(b[1])<<7 + b = b[2:] + } else { + var n int + x, n = decodeVarint(b) + if n == 0 { + return io.ErrUnexpectedEOF + } + b = b[n:] + } + tag := x >> 3 + wire := int(x) & 7 + + // Dispatch on the tag to one of the unmarshal* functions below. + var f unmarshalFieldInfo + if tag < uint64(len(u.dense)) { + f = u.dense[tag] + } else { + f = u.sparse[tag] + } + if fn := f.unmarshal; fn != nil { + var err error + b, err = fn(b, m.offset(f.field), wire) + if err == nil { + reqMask |= f.reqMask + continue + } + if r, ok := err.(*RequiredNotSetError); ok { + // Remember this error, but keep parsing. We need to produce + // a full parse even if a required field is missing. + rnse = r + reqMask |= f.reqMask + continue + } + if err != errInternalBadWireType { + return err + } + // Fragments with bad wire type are treated as unknown fields. + } + + // Unknown tag. + if !u.unrecognized.IsValid() { + // Don't keep unrecognized data; just skip it. + var err error + b, err = skipField(b, wire) + if err != nil { + return err + } + continue + } + // Keep unrecognized data around. + // maybe in extensions, maybe in the unrecognized field. + z := m.offset(u.unrecognized).toBytes() + var emap map[int32]Extension + var e Extension + for _, r := range u.extensionRanges { + if uint64(r.Start) <= tag && tag <= uint64(r.End) { + if u.extensions.IsValid() { + mp := m.offset(u.extensions).toExtensions() + emap = mp.extensionsWrite() + e = emap[int32(tag)] + z = &e.enc + break + } + if u.oldExtensions.IsValid() { + p := m.offset(u.oldExtensions).toOldExtensions() + emap = *p + if emap == nil { + emap = map[int32]Extension{} + *p = emap + } + e = emap[int32(tag)] + z = &e.enc + break + } + panic("no extensions field available") + } + } + + // Use wire type to skip data. + var err error + b0 := b + b, err = skipField(b, wire) + if err != nil { + return err + } + *z = encodeVarint(*z, tag<<3|uint64(wire)) + *z = append(*z, b0[:len(b0)-len(b)]...) + + if emap != nil { + emap[int32(tag)] = e + } + } + if rnse != nil { + // A required field of a submessage/group is missing. Return that error. + return rnse + } + if reqMask != u.reqMask { + // A required field of this message is missing. + for _, n := range u.reqFields { + if reqMask&1 == 0 { + return &RequiredNotSetError{n} + } + reqMask >>= 1 + } + } + return nil +} + +// computeUnmarshalInfo fills in u with information for use +// in unmarshaling protocol buffers of type u.typ. +func (u *unmarshalInfo) computeUnmarshalInfo() { + u.lock.Lock() + defer u.lock.Unlock() + if u.initialized != 0 { + return + } + t := u.typ + n := t.NumField() + + // Set up the "not found" value for the unrecognized byte buffer. + // This is the default for proto3. + u.unrecognized = invalidField + u.extensions = invalidField + u.oldExtensions = invalidField + + // List of the generated type and offset for each oneof field. + type oneofField struct { + ityp reflect.Type // interface type of oneof field + field field // offset in containing message + } + var oneofFields []oneofField + + for i := 0; i < n; i++ { + f := t.Field(i) + if f.Name == "XXX_unrecognized" { + // The byte slice used to hold unrecognized input is special. + if f.Type != reflect.TypeOf(([]byte)(nil)) { + panic("bad type for XXX_unrecognized field: " + f.Type.Name()) + } + u.unrecognized = toField(&f) + continue + } + if f.Name == "XXX_InternalExtensions" { + // Ditto here. + if f.Type != reflect.TypeOf(XXX_InternalExtensions{}) { + panic("bad type for XXX_InternalExtensions field: " + f.Type.Name()) + } + u.extensions = toField(&f) + if f.Tag.Get("protobuf_messageset") == "1" { + u.isMessageSet = true + } + continue + } + if f.Name == "XXX_extensions" { + // An older form of the extensions field. + if f.Type != reflect.TypeOf((map[int32]Extension)(nil)) { + panic("bad type for XXX_extensions field: " + f.Type.Name()) + } + u.oldExtensions = toField(&f) + continue + } + if f.Name == "XXX_NoUnkeyedLiteral" || f.Name == "XXX_sizecache" { + continue + } + + oneof := f.Tag.Get("protobuf_oneof") + if oneof != "" { + oneofFields = append(oneofFields, oneofField{f.Type, toField(&f)}) + // The rest of oneof processing happens below. + continue + } + + tags := f.Tag.Get("protobuf") + tagArray := strings.Split(tags, ",") + if len(tagArray) < 2 { + panic("protobuf tag not enough fields in " + t.Name() + "." + f.Name + ": " + tags) + } + tag, err := strconv.Atoi(tagArray[1]) + if err != nil { + panic("protobuf tag field not an integer: " + tagArray[1]) + } + + name := "" + for _, tag := range tagArray[3:] { + if strings.HasPrefix(tag, "name=") { + name = tag[5:] + } + } + + // Extract unmarshaling function from the field (its type and tags). + unmarshal := fieldUnmarshaler(&f) + + // Required field? + var reqMask uint64 + if tagArray[2] == "req" { + bit := len(u.reqFields) + u.reqFields = append(u.reqFields, name) + reqMask = uint64(1) << uint(bit) + // TODO: if we have more than 64 required fields, we end up + // not verifying that all required fields are present. + // Fix this, perhaps using a count of required fields? + } + + // Store the info in the correct slot in the message. + u.setTag(tag, toField(&f), unmarshal, reqMask) + } + + // Find any types associated with oneof fields. + // TODO: XXX_OneofFuncs returns more info than we need. Get rid of some of it? + fn := reflect.Zero(reflect.PtrTo(t)).MethodByName("XXX_OneofFuncs") + if fn.IsValid() { + res := fn.Call(nil)[3] // last return value from XXX_OneofFuncs: []interface{} + for i := res.Len() - 1; i >= 0; i-- { + v := res.Index(i) // interface{} + tptr := reflect.ValueOf(v.Interface()).Type() // *Msg_X + typ := tptr.Elem() // Msg_X + + f := typ.Field(0) // oneof implementers have one field + baseUnmarshal := fieldUnmarshaler(&f) + tagstr := strings.Split(f.Tag.Get("protobuf"), ",")[1] + tag, err := strconv.Atoi(tagstr) + if err != nil { + panic("protobuf tag field not an integer: " + tagstr) + } + + // Find the oneof field that this struct implements. + // Might take O(n^2) to process all of the oneofs, but who cares. + for _, of := range oneofFields { + if tptr.Implements(of.ityp) { + // We have found the corresponding interface for this struct. + // That lets us know where this struct should be stored + // when we encounter it during unmarshaling. + unmarshal := makeUnmarshalOneof(typ, of.ityp, baseUnmarshal) + u.setTag(tag, of.field, unmarshal, 0) + } + } + } + } + + // Get extension ranges, if any. + fn = reflect.Zero(reflect.PtrTo(t)).MethodByName("ExtensionRangeArray") + if fn.IsValid() { + if !u.extensions.IsValid() && !u.oldExtensions.IsValid() { + panic("a message with extensions, but no extensions field in " + t.Name()) + } + u.extensionRanges = fn.Call(nil)[0].Interface().([]ExtensionRange) + } + + // Explicitly disallow tag 0. This will ensure we flag an error + // when decoding a buffer of all zeros. Without this code, we + // would decode and skip an all-zero buffer of even length. + // [0 0] is [tag=0/wiretype=varint varint-encoded-0]. + u.setTag(0, zeroField, func(b []byte, f pointer, w int) ([]byte, error) { + return nil, fmt.Errorf("proto: %s: illegal tag 0 (wire type %d)", t, w) + }, 0) + + // Set mask for required field check. + u.reqMask = uint64(1)<= 0 && (tag < 16 || tag < 2*n) { // TODO: what are the right numbers here? + for len(u.dense) <= tag { + u.dense = append(u.dense, unmarshalFieldInfo{}) + } + u.dense[tag] = i + return + } + if u.sparse == nil { + u.sparse = map[uint64]unmarshalFieldInfo{} + } + u.sparse[uint64(tag)] = i +} + +// fieldUnmarshaler returns an unmarshaler for the given field. +func fieldUnmarshaler(f *reflect.StructField) unmarshaler { + if f.Type.Kind() == reflect.Map { + return makeUnmarshalMap(f) + } + return typeUnmarshaler(f.Type, f.Tag.Get("protobuf")) +} + +// typeUnmarshaler returns an unmarshaler for the given field type / field tag pair. +func typeUnmarshaler(t reflect.Type, tags string) unmarshaler { + tagArray := strings.Split(tags, ",") + encoding := tagArray[0] + name := "unknown" + for _, tag := range tagArray[3:] { + if strings.HasPrefix(tag, "name=") { + name = tag[5:] + } + } + + // Figure out packaging (pointer, slice, or both) + slice := false + pointer := false + if t.Kind() == reflect.Slice && t.Elem().Kind() != reflect.Uint8 { + slice = true + t = t.Elem() + } + if t.Kind() == reflect.Ptr { + pointer = true + t = t.Elem() + } + + // We'll never have both pointer and slice for basic types. + if pointer && slice && t.Kind() != reflect.Struct { + panic("both pointer and slice for basic type in " + t.Name()) + } + + switch t.Kind() { + case reflect.Bool: + if pointer { + return unmarshalBoolPtr + } + if slice { + return unmarshalBoolSlice + } + return unmarshalBoolValue + case reflect.Int32: + switch encoding { + case "fixed32": + if pointer { + return unmarshalFixedS32Ptr + } + if slice { + return unmarshalFixedS32Slice + } + return unmarshalFixedS32Value + case "varint": + // this could be int32 or enum + if pointer { + return unmarshalInt32Ptr + } + if slice { + return unmarshalInt32Slice + } + return unmarshalInt32Value + case "zigzag32": + if pointer { + return unmarshalSint32Ptr + } + if slice { + return unmarshalSint32Slice + } + return unmarshalSint32Value + } + case reflect.Int64: + switch encoding { + case "fixed64": + if pointer { + return unmarshalFixedS64Ptr + } + if slice { + return unmarshalFixedS64Slice + } + return unmarshalFixedS64Value + case "varint": + if pointer { + return unmarshalInt64Ptr + } + if slice { + return unmarshalInt64Slice + } + return unmarshalInt64Value + case "zigzag64": + if pointer { + return unmarshalSint64Ptr + } + if slice { + return unmarshalSint64Slice + } + return unmarshalSint64Value + } + case reflect.Uint32: + switch encoding { + case "fixed32": + if pointer { + return unmarshalFixed32Ptr + } + if slice { + return unmarshalFixed32Slice + } + return unmarshalFixed32Value + case "varint": + if pointer { + return unmarshalUint32Ptr + } + if slice { + return unmarshalUint32Slice + } + return unmarshalUint32Value + } + case reflect.Uint64: + switch encoding { + case "fixed64": + if pointer { + return unmarshalFixed64Ptr + } + if slice { + return unmarshalFixed64Slice + } + return unmarshalFixed64Value + case "varint": + if pointer { + return unmarshalUint64Ptr + } + if slice { + return unmarshalUint64Slice + } + return unmarshalUint64Value + } + case reflect.Float32: + if pointer { + return unmarshalFloat32Ptr + } + if slice { + return unmarshalFloat32Slice + } + return unmarshalFloat32Value + case reflect.Float64: + if pointer { + return unmarshalFloat64Ptr + } + if slice { + return unmarshalFloat64Slice + } + return unmarshalFloat64Value + case reflect.Map: + panic("map type in typeUnmarshaler in " + t.Name()) + case reflect.Slice: + if pointer { + panic("bad pointer in slice case in " + t.Name()) + } + if slice { + return unmarshalBytesSlice + } + return unmarshalBytesValue + case reflect.String: + if pointer { + return unmarshalStringPtr + } + if slice { + return unmarshalStringSlice + } + return unmarshalStringValue + case reflect.Struct: + // message or group field + if !pointer { + panic(fmt.Sprintf("message/group field %s:%s without pointer", t, encoding)) + } + switch encoding { + case "bytes": + if slice { + return makeUnmarshalMessageSlicePtr(getUnmarshalInfo(t), name) + } + return makeUnmarshalMessagePtr(getUnmarshalInfo(t), name) + case "group": + if slice { + return makeUnmarshalGroupSlicePtr(getUnmarshalInfo(t), name) + } + return makeUnmarshalGroupPtr(getUnmarshalInfo(t), name) + } + } + panic(fmt.Sprintf("unmarshaler not found type:%s encoding:%s", t, encoding)) +} + +// Below are all the unmarshalers for individual fields of various types. + +func unmarshalInt64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x) + *f.toInt64() = v + return b, nil +} + +func unmarshalInt64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x) + *f.toInt64Ptr() = &v + return b, nil +} + +func unmarshalInt64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x) + s := f.toInt64Slice() + *s = append(*s, v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x) + s := f.toInt64Slice() + *s = append(*s, v) + return b, nil +} + +func unmarshalSint64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x>>1) ^ int64(x)<<63>>63 + *f.toInt64() = v + return b, nil +} + +func unmarshalSint64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x>>1) ^ int64(x)<<63>>63 + *f.toInt64Ptr() = &v + return b, nil +} + +func unmarshalSint64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x>>1) ^ int64(x)<<63>>63 + s := f.toInt64Slice() + *s = append(*s, v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x>>1) ^ int64(x)<<63>>63 + s := f.toInt64Slice() + *s = append(*s, v) + return b, nil +} + +func unmarshalUint64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint64(x) + *f.toUint64() = v + return b, nil +} + +func unmarshalUint64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint64(x) + *f.toUint64Ptr() = &v + return b, nil +} + +func unmarshalUint64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint64(x) + s := f.toUint64Slice() + *s = append(*s, v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint64(x) + s := f.toUint64Slice() + *s = append(*s, v) + return b, nil +} + +func unmarshalInt32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x) + *f.toInt32() = v + return b, nil +} + +func unmarshalInt32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x) + f.setInt32Ptr(v) + return b, nil +} + +func unmarshalInt32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x) + f.appendInt32Slice(v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x) + f.appendInt32Slice(v) + return b, nil +} + +func unmarshalSint32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x>>1) ^ int32(x)<<31>>31 + *f.toInt32() = v + return b, nil +} + +func unmarshalSint32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x>>1) ^ int32(x)<<31>>31 + f.setInt32Ptr(v) + return b, nil +} + +func unmarshalSint32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x>>1) ^ int32(x)<<31>>31 + f.appendInt32Slice(v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x>>1) ^ int32(x)<<31>>31 + f.appendInt32Slice(v) + return b, nil +} + +func unmarshalUint32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint32(x) + *f.toUint32() = v + return b, nil +} + +func unmarshalUint32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint32(x) + *f.toUint32Ptr() = &v + return b, nil +} + +func unmarshalUint32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint32(x) + s := f.toUint32Slice() + *s = append(*s, v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint32(x) + s := f.toUint32Slice() + *s = append(*s, v) + return b, nil +} + +func unmarshalFixed64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + *f.toUint64() = v + return b[8:], nil +} + +func unmarshalFixed64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + *f.toUint64Ptr() = &v + return b[8:], nil +} + +func unmarshalFixed64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + s := f.toUint64Slice() + *s = append(*s, v) + b = b[8:] + } + return res, nil + } + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + s := f.toUint64Slice() + *s = append(*s, v) + return b[8:], nil +} + +func unmarshalFixedS64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56 + *f.toInt64() = v + return b[8:], nil +} + +func unmarshalFixedS64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56 + *f.toInt64Ptr() = &v + return b[8:], nil +} + +func unmarshalFixedS64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56 + s := f.toInt64Slice() + *s = append(*s, v) + b = b[8:] + } + return res, nil + } + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56 + s := f.toInt64Slice() + *s = append(*s, v) + return b[8:], nil +} + +func unmarshalFixed32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 + *f.toUint32() = v + return b[4:], nil +} + +func unmarshalFixed32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 + *f.toUint32Ptr() = &v + return b[4:], nil +} + +func unmarshalFixed32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 + s := f.toUint32Slice() + *s = append(*s, v) + b = b[4:] + } + return res, nil + } + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 + s := f.toUint32Slice() + *s = append(*s, v) + return b[4:], nil +} + +func unmarshalFixedS32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24 + *f.toInt32() = v + return b[4:], nil +} + +func unmarshalFixedS32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24 + f.setInt32Ptr(v) + return b[4:], nil +} + +func unmarshalFixedS32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24 + f.appendInt32Slice(v) + b = b[4:] + } + return res, nil + } + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24 + f.appendInt32Slice(v) + return b[4:], nil +} + +func unmarshalBoolValue(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + // Note: any length varint is allowed, even though any sane + // encoder will use one byte. + // See https://github.com/golang/protobuf/issues/76 + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + // TODO: check if x>1? Tests seem to indicate no. + v := x != 0 + *f.toBool() = v + return b[n:], nil +} + +func unmarshalBoolPtr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + v := x != 0 + *f.toBoolPtr() = &v + return b[n:], nil +} + +func unmarshalBoolSlice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + v := x != 0 + s := f.toBoolSlice() + *s = append(*s, v) + b = b[n:] + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + v := x != 0 + s := f.toBoolSlice() + *s = append(*s, v) + return b[n:], nil +} + +func unmarshalFloat64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56) + *f.toFloat64() = v + return b[8:], nil +} + +func unmarshalFloat64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56) + *f.toFloat64Ptr() = &v + return b[8:], nil +} + +func unmarshalFloat64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56) + s := f.toFloat64Slice() + *s = append(*s, v) + b = b[8:] + } + return res, nil + } + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56) + s := f.toFloat64Slice() + *s = append(*s, v) + return b[8:], nil +} + +func unmarshalFloat32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24) + *f.toFloat32() = v + return b[4:], nil +} + +func unmarshalFloat32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24) + *f.toFloat32Ptr() = &v + return b[4:], nil +} + +func unmarshalFloat32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24) + s := f.toFloat32Slice() + *s = append(*s, v) + b = b[4:] + } + return res, nil + } + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24) + s := f.toFloat32Slice() + *s = append(*s, v) + return b[4:], nil +} + +func unmarshalStringValue(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + v := string(b[:x]) + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + *f.toString() = v + return b[x:], nil +} + +func unmarshalStringPtr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + v := string(b[:x]) + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + *f.toStringPtr() = &v + return b[x:], nil +} + +func unmarshalStringSlice(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + v := string(b[:x]) + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + s := f.toStringSlice() + *s = append(*s, v) + return b[x:], nil +} + +var emptyBuf [0]byte + +func unmarshalBytesValue(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + // The use of append here is a trick which avoids the zeroing + // that would be required if we used a make/copy pair. + // We append to emptyBuf instead of nil because we want + // a non-nil result even when the length is 0. + v := append(emptyBuf[:], b[:x]...) + *f.toBytes() = v + return b[x:], nil +} + +func unmarshalBytesSlice(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + v := append(emptyBuf[:], b[:x]...) + s := f.toBytesSlice() + *s = append(*s, v) + return b[x:], nil +} + +func makeUnmarshalMessagePtr(sub *unmarshalInfo, name string) unmarshaler { + return func(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + // First read the message field to see if something is there. + // The semantics of multiple submessages are weird. Instead of + // the last one winning (as it is for all other fields), multiple + // submessages are merged. + v := f.getPointer() + if v.isNil() { + v = valToPointer(reflect.New(sub.typ)) + f.setPointer(v) + } + err := sub.unmarshal(v, b[:x]) + if err != nil { + if r, ok := err.(*RequiredNotSetError); ok { + r.field = name + "." + r.field + } else { + return nil, err + } + } + return b[x:], err + } +} + +func makeUnmarshalMessageSlicePtr(sub *unmarshalInfo, name string) unmarshaler { + return func(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + v := valToPointer(reflect.New(sub.typ)) + err := sub.unmarshal(v, b[:x]) + if err != nil { + if r, ok := err.(*RequiredNotSetError); ok { + r.field = name + "." + r.field + } else { + return nil, err + } + } + f.appendPointer(v) + return b[x:], err + } +} + +func makeUnmarshalGroupPtr(sub *unmarshalInfo, name string) unmarshaler { + return func(b []byte, f pointer, w int) ([]byte, error) { + if w != WireStartGroup { + return b, errInternalBadWireType + } + x, y := findEndGroup(b) + if x < 0 { + return nil, io.ErrUnexpectedEOF + } + v := f.getPointer() + if v.isNil() { + v = valToPointer(reflect.New(sub.typ)) + f.setPointer(v) + } + err := sub.unmarshal(v, b[:x]) + if err != nil { + if r, ok := err.(*RequiredNotSetError); ok { + r.field = name + "." + r.field + } else { + return nil, err + } + } + return b[y:], err + } +} + +func makeUnmarshalGroupSlicePtr(sub *unmarshalInfo, name string) unmarshaler { + return func(b []byte, f pointer, w int) ([]byte, error) { + if w != WireStartGroup { + return b, errInternalBadWireType + } + x, y := findEndGroup(b) + if x < 0 { + return nil, io.ErrUnexpectedEOF + } + v := valToPointer(reflect.New(sub.typ)) + err := sub.unmarshal(v, b[:x]) + if err != nil { + if r, ok := err.(*RequiredNotSetError); ok { + r.field = name + "." + r.field + } else { + return nil, err + } + } + f.appendPointer(v) + return b[y:], err + } +} + +func makeUnmarshalMap(f *reflect.StructField) unmarshaler { + t := f.Type + kt := t.Key() + vt := t.Elem() + unmarshalKey := typeUnmarshaler(kt, f.Tag.Get("protobuf_key")) + unmarshalVal := typeUnmarshaler(vt, f.Tag.Get("protobuf_val")) + return func(b []byte, f pointer, w int) ([]byte, error) { + // The map entry is a submessage. Figure out how big it is. + if w != WireBytes { + return nil, fmt.Errorf("proto: bad wiretype for map field: got %d want %d", w, WireBytes) + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + r := b[x:] // unused data to return + b = b[:x] // data for map entry + + // Note: we could use #keys * #values ~= 200 functions + // to do map decoding without reflection. Probably not worth it. + // Maps will be somewhat slow. Oh well. + + // Read key and value from data. + k := reflect.New(kt) + v := reflect.New(vt) + for len(b) > 0 { + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + wire := int(x) & 7 + b = b[n:] + + var err error + switch x >> 3 { + case 1: + b, err = unmarshalKey(b, valToPointer(k), wire) + case 2: + b, err = unmarshalVal(b, valToPointer(v), wire) + default: + err = errInternalBadWireType // skip unknown tag + } + + if err == nil { + continue + } + if err != errInternalBadWireType { + return nil, err + } + + // Skip past unknown fields. + b, err = skipField(b, wire) + if err != nil { + return nil, err + } + } + + // Get map, allocate if needed. + m := f.asPointerTo(t).Elem() // an addressable map[K]T + if m.IsNil() { + m.Set(reflect.MakeMap(t)) + } + + // Insert into map. + m.SetMapIndex(k.Elem(), v.Elem()) + + return r, nil + } +} + +// makeUnmarshalOneof makes an unmarshaler for oneof fields. +// for: +// message Msg { +// oneof F { +// int64 X = 1; +// float64 Y = 2; +// } +// } +// typ is the type of the concrete entry for a oneof case (e.g. Msg_X). +// ityp is the interface type of the oneof field (e.g. isMsg_F). +// unmarshal is the unmarshaler for the base type of the oneof case (e.g. int64). +// Note that this function will be called once for each case in the oneof. +func makeUnmarshalOneof(typ, ityp reflect.Type, unmarshal unmarshaler) unmarshaler { + sf := typ.Field(0) + field0 := toField(&sf) + return func(b []byte, f pointer, w int) ([]byte, error) { + // Allocate holder for value. + v := reflect.New(typ) + + // Unmarshal data into holder. + // We unmarshal into the first field of the holder object. + var err error + b, err = unmarshal(b, valToPointer(v).offset(field0), w) + if err != nil { + return nil, err + } + + // Write pointer to holder into target field. + f.asPointerTo(ityp).Elem().Set(v) + + return b, nil + } +} + +// Error used by decode internally. +var errInternalBadWireType = errors.New("proto: internal error: bad wiretype") + +// skipField skips past a field of type wire and returns the remaining bytes. +func skipField(b []byte, wire int) ([]byte, error) { + switch wire { + case WireVarint: + _, k := decodeVarint(b) + if k == 0 { + return b, io.ErrUnexpectedEOF + } + b = b[k:] + case WireFixed32: + if len(b) < 4 { + return b, io.ErrUnexpectedEOF + } + b = b[4:] + case WireFixed64: + if len(b) < 8 { + return b, io.ErrUnexpectedEOF + } + b = b[8:] + case WireBytes: + m, k := decodeVarint(b) + if k == 0 || uint64(len(b)-k) < m { + return b, io.ErrUnexpectedEOF + } + b = b[uint64(k)+m:] + case WireStartGroup: + _, i := findEndGroup(b) + if i == -1 { + return b, io.ErrUnexpectedEOF + } + b = b[i:] + default: + return b, fmt.Errorf("proto: can't skip unknown wire type %d", wire) + } + return b, nil +} + +// findEndGroup finds the index of the next EndGroup tag. +// Groups may be nested, so the "next" EndGroup tag is the first +// unpaired EndGroup. +// findEndGroup returns the indexes of the start and end of the EndGroup tag. +// Returns (-1,-1) if it can't find one. +func findEndGroup(b []byte) (int, int) { + depth := 1 + i := 0 + for { + x, n := decodeVarint(b[i:]) + if n == 0 { + return -1, -1 + } + j := i + i += n + switch x & 7 { + case WireVarint: + _, k := decodeVarint(b[i:]) + if k == 0 { + return -1, -1 + } + i += k + case WireFixed32: + if len(b)-4 < i { + return -1, -1 + } + i += 4 + case WireFixed64: + if len(b)-8 < i { + return -1, -1 + } + i += 8 + case WireBytes: + m, k := decodeVarint(b[i:]) + if k == 0 { + return -1, -1 + } + i += k + if uint64(len(b)-i) < m { + return -1, -1 + } + i += int(m) + case WireStartGroup: + depth++ + case WireEndGroup: + depth-- + if depth == 0 { + return j, i + } + default: + return -1, -1 + } + } +} + +// encodeVarint appends a varint-encoded integer to b and returns the result. +func encodeVarint(b []byte, x uint64) []byte { + for x >= 1<<7 { + b = append(b, byte(x&0x7f|0x80)) + x >>= 7 + } + return append(b, byte(x)) +} + +// decodeVarint reads a varint-encoded integer from b. +// Returns the decoded integer and the number of bytes read. +// If there is an error, it returns 0,0. +func decodeVarint(b []byte) (uint64, int) { + var x, y uint64 + if len(b) <= 0 { + goto bad + } + x = uint64(b[0]) + if x < 0x80 { + return x, 1 + } + x -= 0x80 + + if len(b) <= 1 { + goto bad + } + y = uint64(b[1]) + x += y << 7 + if y < 0x80 { + return x, 2 + } + x -= 0x80 << 7 + + if len(b) <= 2 { + goto bad + } + y = uint64(b[2]) + x += y << 14 + if y < 0x80 { + return x, 3 + } + x -= 0x80 << 14 + + if len(b) <= 3 { + goto bad + } + y = uint64(b[3]) + x += y << 21 + if y < 0x80 { + return x, 4 + } + x -= 0x80 << 21 + + if len(b) <= 4 { + goto bad + } + y = uint64(b[4]) + x += y << 28 + if y < 0x80 { + return x, 5 + } + x -= 0x80 << 28 + + if len(b) <= 5 { + goto bad + } + y = uint64(b[5]) + x += y << 35 + if y < 0x80 { + return x, 6 + } + x -= 0x80 << 35 + + if len(b) <= 6 { + goto bad + } + y = uint64(b[6]) + x += y << 42 + if y < 0x80 { + return x, 7 + } + x -= 0x80 << 42 + + if len(b) <= 7 { + goto bad + } + y = uint64(b[7]) + x += y << 49 + if y < 0x80 { + return x, 8 + } + x -= 0x80 << 49 + + if len(b) <= 8 { + goto bad + } + y = uint64(b[8]) + x += y << 56 + if y < 0x80 { + return x, 9 + } + x -= 0x80 << 56 + + if len(b) <= 9 { + goto bad + } + y = uint64(b[9]) + x += y << 63 + if y < 2 { + return x, 10 + } + +bad: + return 0, 0 +} diff --git a/vendor/github.com/golang/protobuf/proto/text.go b/vendor/github.com/golang/protobuf/proto/text.go index 965876bf033..2205fdaadf8 100644 --- a/vendor/github.com/golang/protobuf/proto/text.go +++ b/vendor/github.com/golang/protobuf/proto/text.go @@ -50,7 +50,6 @@ import ( var ( newline = []byte("\n") spaces = []byte(" ") - gtNewline = []byte(">\n") endBraceNewline = []byte("}\n") backslashN = []byte{'\\', 'n'} backslashR = []byte{'\\', 'r'} @@ -170,11 +169,6 @@ func writeName(w *textWriter, props *Properties) error { return nil } -// raw is the interface satisfied by RawMessage. -type raw interface { - Bytes() []byte -} - func requiresQuotes(u string) bool { // When type URL contains any characters except [0-9A-Za-z./\-]*, it must be quoted. for _, ch := range u { @@ -269,6 +263,10 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { props := sprops.Prop[i] name := st.Field(i).Name + if name == "XXX_NoUnkeyedLiteral" { + continue + } + if strings.HasPrefix(name, "XXX_") { // There are two XXX_ fields: // XXX_unrecognized []byte @@ -436,12 +434,6 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { return err } } - if b, ok := fv.Interface().(raw); ok { - if err := writeRaw(w, b.Bytes()); err != nil { - return err - } - continue - } // Enums have a String method, so writeAny will work fine. if err := tm.writeAny(w, fv, props); err != nil { @@ -455,7 +447,7 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { // Extensions (the XXX_extensions field). pv := sv.Addr() - if _, ok := extendable(pv.Interface()); ok { + if _, err := extendable(pv.Interface()); err == nil { if err := tm.writeExtensions(w, pv); err != nil { return err } @@ -464,27 +456,6 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { return nil } -// writeRaw writes an uninterpreted raw message. -func writeRaw(w *textWriter, b []byte) error { - if err := w.WriteByte('<'); err != nil { - return err - } - if !w.compact { - if err := w.WriteByte('\n'); err != nil { - return err - } - } - w.indent() - if err := writeUnknownStruct(w, b); err != nil { - return err - } - w.unindent() - if err := w.WriteByte('>'); err != nil { - return err - } - return nil -} - // writeAny writes an arbitrary field. func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Properties) error { v = reflect.Indirect(v) @@ -535,6 +506,19 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert } } w.indent() + if v.CanAddr() { + // Calling v.Interface on a struct causes the reflect package to + // copy the entire struct. This is racy with the new Marshaler + // since we atomically update the XXX_sizecache. + // + // Thus, we retrieve a pointer to the struct if possible to avoid + // a race since v.Interface on the pointer doesn't copy the struct. + // + // If v is not addressable, then we are not worried about a race + // since it implies that the binary Marshaler cannot possibly be + // mutating this value. + v = v.Addr() + } if etm, ok := v.Interface().(encoding.TextMarshaler); ok { text, err := etm.MarshalText() if err != nil { @@ -543,8 +527,13 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert if _, err = w.Write(text); err != nil { return err } - } else if err := tm.writeStruct(w, v); err != nil { - return err + } else { + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + if err := tm.writeStruct(w, v); err != nil { + return err + } } w.unindent() if err := w.WriteByte(ket); err != nil { diff --git a/vendor/github.com/golang/protobuf/proto/text_parser.go b/vendor/github.com/golang/protobuf/proto/text_parser.go index 61f83c1e10f..0685bae36d5 100644 --- a/vendor/github.com/golang/protobuf/proto/text_parser.go +++ b/vendor/github.com/golang/protobuf/proto/text_parser.go @@ -206,7 +206,6 @@ func (p *textParser) advance() { var ( errBadUTF8 = errors.New("proto: bad UTF-8") - errBadHex = errors.New("proto: bad hexadecimal") ) func unquoteC(s string, quote rune) (string, error) { @@ -277,60 +276,47 @@ func unescape(s string) (ch string, tail string, err error) { return "?", s, nil // trigraph workaround case '\'', '"', '\\': return string(r), s, nil - case '0', '1', '2', '3', '4', '5', '6', '7', 'x', 'X': + case '0', '1', '2', '3', '4', '5', '6', '7': if len(s) < 2 { return "", "", fmt.Errorf(`\%c requires 2 following digits`, r) } - base := 8 - ss := s[:2] + ss := string(r) + s[:2] s = s[2:] - if r == 'x' || r == 'X' { - base = 16 - } else { - ss = string(r) + ss - } - i, err := strconv.ParseUint(ss, base, 8) + i, err := strconv.ParseUint(ss, 8, 8) if err != nil { - return "", "", err + return "", "", fmt.Errorf(`\%s contains non-octal digits`, ss) } return string([]byte{byte(i)}), s, nil - case 'u', 'U': - n := 4 - if r == 'U' { + case 'x', 'X', 'u', 'U': + var n int + switch r { + case 'x', 'X': + n = 2 + case 'u': + n = 4 + case 'U': n = 8 } if len(s) < n { - return "", "", fmt.Errorf(`\%c requires %d digits`, r, n) - } - - bs := make([]byte, n/2) - for i := 0; i < n; i += 2 { - a, ok1 := unhex(s[i]) - b, ok2 := unhex(s[i+1]) - if !ok1 || !ok2 { - return "", "", errBadHex - } - bs[i/2] = a<<4 | b + return "", "", fmt.Errorf(`\%c requires %d following digits`, r, n) } + ss := s[:n] s = s[n:] - return string(bs), s, nil + i, err := strconv.ParseUint(ss, 16, 64) + if err != nil { + return "", "", fmt.Errorf(`\%c%s contains non-hexadecimal digits`, r, ss) + } + if r == 'x' || r == 'X' { + return string([]byte{byte(i)}), s, nil + } + if i > utf8.MaxRune { + return "", "", fmt.Errorf(`\%c%s is not a valid Unicode code point`, r, ss) + } + return string(i), s, nil } return "", "", fmt.Errorf(`unknown escape \%c`, r) } -// Adapted from src/pkg/strconv/quote.go. -func unhex(b byte) (v byte, ok bool) { - switch { - case '0' <= b && b <= '9': - return b - '0', true - case 'a' <= b && b <= 'f': - return b - 'a' + 10, true - case 'A' <= b && b <= 'F': - return b - 'A' + 10, true - } - return 0, false -} - // Back off the parser by one token. Can only be done between calls to next(). // It makes the next advance() a no-op. func (p *textParser) back() { p.backed = true } @@ -728,6 +714,9 @@ func (p *textParser) consumeExtName() (string, error) { if tok.err != nil { return "", p.errorf("unrecognized type_url or extension name: %s", tok.err) } + if p.done && tok.value != "]" { + return "", p.errorf("unclosed type_url or extension name") + } } return strings.Join(parts, ""), nil } @@ -883,13 +872,9 @@ func (p *textParser) readAny(v reflect.Value, props *Properties) error { // UnmarshalText returns *RequiredNotSetError. func UnmarshalText(s string, pb Message) error { if um, ok := pb.(encoding.TextUnmarshaler); ok { - err := um.UnmarshalText([]byte(s)) - return err + return um.UnmarshalText([]byte(s)) } pb.Reset() v := reflect.ValueOf(pb) - if pe := newTextParser(s).readStruct(v.Elem(), ""); pe != nil { - return pe - } - return nil + return newTextParser(s).readStruct(v.Elem(), "") } diff --git a/vendor/golang.org/x/net/context/context.go b/vendor/golang.org/x/net/context/context.go index f143ed6a1eb..a3c021d3f88 100644 --- a/vendor/golang.org/x/net/context/context.go +++ b/vendor/golang.org/x/net/context/context.go @@ -5,6 +5,8 @@ // Package context defines the Context type, which carries deadlines, // cancelation signals, and other request-scoped values across API boundaries // and between processes. +// As of Go 1.7 this package is available in the standard library under the +// name context. https://golang.org/pkg/context. // // Incoming requests to a server should create a Context, and outgoing calls to // servers should accept a Context. The chain of function calls between must @@ -36,103 +38,6 @@ // Contexts. package context // import "golang.org/x/net/context" -import "time" - -// A Context carries a deadline, a cancelation signal, and other values across -// API boundaries. -// -// Context's methods may be called by multiple goroutines simultaneously. -type Context interface { - // Deadline returns the time when work done on behalf of this context - // should be canceled. Deadline returns ok==false when no deadline is - // set. Successive calls to Deadline return the same results. - Deadline() (deadline time.Time, ok bool) - - // Done returns a channel that's closed when work done on behalf of this - // context should be canceled. Done may return nil if this context can - // never be canceled. Successive calls to Done return the same value. - // - // WithCancel arranges for Done to be closed when cancel is called; - // WithDeadline arranges for Done to be closed when the deadline - // expires; WithTimeout arranges for Done to be closed when the timeout - // elapses. - // - // Done is provided for use in select statements: - // - // // Stream generates values with DoSomething and sends them to out - // // until DoSomething returns an error or ctx.Done is closed. - // func Stream(ctx context.Context, out chan<- Value) error { - // for { - // v, err := DoSomething(ctx) - // if err != nil { - // return err - // } - // select { - // case <-ctx.Done(): - // return ctx.Err() - // case out <- v: - // } - // } - // } - // - // See http://blog.golang.org/pipelines for more examples of how to use - // a Done channel for cancelation. - Done() <-chan struct{} - - // Err returns a non-nil error value after Done is closed. Err returns - // Canceled if the context was canceled or DeadlineExceeded if the - // context's deadline passed. No other values for Err are defined. - // After Done is closed, successive calls to Err return the same value. - Err() error - - // Value returns the value associated with this context for key, or nil - // if no value is associated with key. Successive calls to Value with - // the same key returns the same result. - // - // Use context values only for request-scoped data that transits - // processes and API boundaries, not for passing optional parameters to - // functions. - // - // A key identifies a specific value in a Context. Functions that wish - // to store values in Context typically allocate a key in a global - // variable then use that key as the argument to context.WithValue and - // Context.Value. A key can be any type that supports equality; - // packages should define keys as an unexported type to avoid - // collisions. - // - // Packages that define a Context key should provide type-safe accessors - // for the values stores using that key: - // - // // Package user defines a User type that's stored in Contexts. - // package user - // - // import "golang.org/x/net/context" - // - // // User is the type of value stored in the Contexts. - // type User struct {...} - // - // // key is an unexported type for keys defined in this package. - // // This prevents collisions with keys defined in other packages. - // type key int - // - // // userKey is the key for user.User values in Contexts. It is - // // unexported; clients use user.NewContext and user.FromContext - // // instead of using this key directly. - // var userKey key = 0 - // - // // NewContext returns a new Context that carries value u. - // func NewContext(ctx context.Context, u *User) context.Context { - // return context.WithValue(ctx, userKey, u) - // } - // - // // FromContext returns the User value stored in ctx, if any. - // func FromContext(ctx context.Context) (*User, bool) { - // u, ok := ctx.Value(userKey).(*User) - // return u, ok - // } - Value(key interface{}) interface{} -} - // Background returns a non-nil, empty Context. It is never canceled, has no // values, and has no deadline. It is typically used by the main function, // initialization, and tests, and as the top-level Context for incoming @@ -149,8 +54,3 @@ func Background() Context { func TODO() Context { return todo } - -// A CancelFunc tells an operation to abandon its work. -// A CancelFunc does not wait for the work to stop. -// After the first call, subsequent calls to a CancelFunc do nothing. -type CancelFunc func() diff --git a/vendor/golang.org/x/net/context/go19.go b/vendor/golang.org/x/net/context/go19.go new file mode 100644 index 00000000000..d88bd1db127 --- /dev/null +++ b/vendor/golang.org/x/net/context/go19.go @@ -0,0 +1,20 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 + +package context + +import "context" // standard library's context, as of Go 1.7 + +// A Context carries a deadline, a cancelation signal, and other values across +// API boundaries. +// +// Context's methods may be called by multiple goroutines simultaneously. +type Context = context.Context + +// A CancelFunc tells an operation to abandon its work. +// A CancelFunc does not wait for the work to stop. +// After the first call, subsequent calls to a CancelFunc do nothing. +type CancelFunc = context.CancelFunc diff --git a/vendor/golang.org/x/net/context/pre_go19.go b/vendor/golang.org/x/net/context/pre_go19.go new file mode 100644 index 00000000000..b105f80be4f --- /dev/null +++ b/vendor/golang.org/x/net/context/pre_go19.go @@ -0,0 +1,109 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.9 + +package context + +import "time" + +// A Context carries a deadline, a cancelation signal, and other values across +// API boundaries. +// +// Context's methods may be called by multiple goroutines simultaneously. +type Context interface { + // Deadline returns the time when work done on behalf of this context + // should be canceled. Deadline returns ok==false when no deadline is + // set. Successive calls to Deadline return the same results. + Deadline() (deadline time.Time, ok bool) + + // Done returns a channel that's closed when work done on behalf of this + // context should be canceled. Done may return nil if this context can + // never be canceled. Successive calls to Done return the same value. + // + // WithCancel arranges for Done to be closed when cancel is called; + // WithDeadline arranges for Done to be closed when the deadline + // expires; WithTimeout arranges for Done to be closed when the timeout + // elapses. + // + // Done is provided for use in select statements: + // + // // Stream generates values with DoSomething and sends them to out + // // until DoSomething returns an error or ctx.Done is closed. + // func Stream(ctx context.Context, out chan<- Value) error { + // for { + // v, err := DoSomething(ctx) + // if err != nil { + // return err + // } + // select { + // case <-ctx.Done(): + // return ctx.Err() + // case out <- v: + // } + // } + // } + // + // See http://blog.golang.org/pipelines for more examples of how to use + // a Done channel for cancelation. + Done() <-chan struct{} + + // Err returns a non-nil error value after Done is closed. Err returns + // Canceled if the context was canceled or DeadlineExceeded if the + // context's deadline passed. No other values for Err are defined. + // After Done is closed, successive calls to Err return the same value. + Err() error + + // Value returns the value associated with this context for key, or nil + // if no value is associated with key. Successive calls to Value with + // the same key returns the same result. + // + // Use context values only for request-scoped data that transits + // processes and API boundaries, not for passing optional parameters to + // functions. + // + // A key identifies a specific value in a Context. Functions that wish + // to store values in Context typically allocate a key in a global + // variable then use that key as the argument to context.WithValue and + // Context.Value. A key can be any type that supports equality; + // packages should define keys as an unexported type to avoid + // collisions. + // + // Packages that define a Context key should provide type-safe accessors + // for the values stores using that key: + // + // // Package user defines a User type that's stored in Contexts. + // package user + // + // import "golang.org/x/net/context" + // + // // User is the type of value stored in the Contexts. + // type User struct {...} + // + // // key is an unexported type for keys defined in this package. + // // This prevents collisions with keys defined in other packages. + // type key int + // + // // userKey is the key for user.User values in Contexts. It is + // // unexported; clients use user.NewContext and user.FromContext + // // instead of using this key directly. + // var userKey key = 0 + // + // // NewContext returns a new Context that carries value u. + // func NewContext(ctx context.Context, u *User) context.Context { + // return context.WithValue(ctx, userKey, u) + // } + // + // // FromContext returns the User value stored in ctx, if any. + // func FromContext(ctx context.Context) (*User, bool) { + // u, ok := ctx.Value(userKey).(*User) + // return u, ok + // } + Value(key interface{}) interface{} +} + +// A CancelFunc tells an operation to abandon its work. +// A CancelFunc does not wait for the work to stop. +// After the first call, subsequent calls to a CancelFunc do nothing. +type CancelFunc func() diff --git a/vendor/golang.org/x/net/http2/ciphers.go b/vendor/golang.org/x/net/http2/ciphers.go new file mode 100644 index 00000000000..c9a0cf3b422 --- /dev/null +++ b/vendor/golang.org/x/net/http2/ciphers.go @@ -0,0 +1,641 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http2 + +// A list of the possible cipher suite ids. Taken from +// https://www.iana.org/assignments/tls-parameters/tls-parameters.txt + +const ( + cipher_TLS_NULL_WITH_NULL_NULL uint16 = 0x0000 + cipher_TLS_RSA_WITH_NULL_MD5 uint16 = 0x0001 + cipher_TLS_RSA_WITH_NULL_SHA uint16 = 0x0002 + cipher_TLS_RSA_EXPORT_WITH_RC4_40_MD5 uint16 = 0x0003 + cipher_TLS_RSA_WITH_RC4_128_MD5 uint16 = 0x0004 + cipher_TLS_RSA_WITH_RC4_128_SHA uint16 = 0x0005 + cipher_TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5 uint16 = 0x0006 + cipher_TLS_RSA_WITH_IDEA_CBC_SHA uint16 = 0x0007 + cipher_TLS_RSA_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0008 + cipher_TLS_RSA_WITH_DES_CBC_SHA uint16 = 0x0009 + cipher_TLS_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0x000A + cipher_TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x000B + cipher_TLS_DH_DSS_WITH_DES_CBC_SHA uint16 = 0x000C + cipher_TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA uint16 = 0x000D + cipher_TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x000E + cipher_TLS_DH_RSA_WITH_DES_CBC_SHA uint16 = 0x000F + cipher_TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0x0010 + cipher_TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0011 + cipher_TLS_DHE_DSS_WITH_DES_CBC_SHA uint16 = 0x0012 + cipher_TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA uint16 = 0x0013 + cipher_TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0014 + cipher_TLS_DHE_RSA_WITH_DES_CBC_SHA uint16 = 0x0015 + cipher_TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0x0016 + cipher_TLS_DH_anon_EXPORT_WITH_RC4_40_MD5 uint16 = 0x0017 + cipher_TLS_DH_anon_WITH_RC4_128_MD5 uint16 = 0x0018 + cipher_TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0019 + cipher_TLS_DH_anon_WITH_DES_CBC_SHA uint16 = 0x001A + cipher_TLS_DH_anon_WITH_3DES_EDE_CBC_SHA uint16 = 0x001B + // Reserved uint16 = 0x001C-1D + cipher_TLS_KRB5_WITH_DES_CBC_SHA uint16 = 0x001E + cipher_TLS_KRB5_WITH_3DES_EDE_CBC_SHA uint16 = 0x001F + cipher_TLS_KRB5_WITH_RC4_128_SHA uint16 = 0x0020 + cipher_TLS_KRB5_WITH_IDEA_CBC_SHA uint16 = 0x0021 + cipher_TLS_KRB5_WITH_DES_CBC_MD5 uint16 = 0x0022 + cipher_TLS_KRB5_WITH_3DES_EDE_CBC_MD5 uint16 = 0x0023 + cipher_TLS_KRB5_WITH_RC4_128_MD5 uint16 = 0x0024 + cipher_TLS_KRB5_WITH_IDEA_CBC_MD5 uint16 = 0x0025 + cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA uint16 = 0x0026 + cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA uint16 = 0x0027 + cipher_TLS_KRB5_EXPORT_WITH_RC4_40_SHA uint16 = 0x0028 + cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5 uint16 = 0x0029 + cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5 uint16 = 0x002A + cipher_TLS_KRB5_EXPORT_WITH_RC4_40_MD5 uint16 = 0x002B + cipher_TLS_PSK_WITH_NULL_SHA uint16 = 0x002C + cipher_TLS_DHE_PSK_WITH_NULL_SHA uint16 = 0x002D + cipher_TLS_RSA_PSK_WITH_NULL_SHA uint16 = 0x002E + cipher_TLS_RSA_WITH_AES_128_CBC_SHA uint16 = 0x002F + cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA uint16 = 0x0030 + cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA uint16 = 0x0031 + cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA uint16 = 0x0032 + cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA uint16 = 0x0033 + cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA uint16 = 0x0034 + cipher_TLS_RSA_WITH_AES_256_CBC_SHA uint16 = 0x0035 + cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA uint16 = 0x0036 + cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA uint16 = 0x0037 + cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA uint16 = 0x0038 + cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA uint16 = 0x0039 + cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA uint16 = 0x003A + cipher_TLS_RSA_WITH_NULL_SHA256 uint16 = 0x003B + cipher_TLS_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0x003C + cipher_TLS_RSA_WITH_AES_256_CBC_SHA256 uint16 = 0x003D + cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA256 uint16 = 0x003E + cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0x003F + cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 uint16 = 0x0040 + cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0041 + cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0042 + cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0043 + cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0044 + cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0045 + cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0046 + // Reserved uint16 = 0x0047-4F + // Reserved uint16 = 0x0050-58 + // Reserved uint16 = 0x0059-5C + // Unassigned uint16 = 0x005D-5F + // Reserved uint16 = 0x0060-66 + cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0x0067 + cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA256 uint16 = 0x0068 + cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA256 uint16 = 0x0069 + cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 uint16 = 0x006A + cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 uint16 = 0x006B + cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA256 uint16 = 0x006C + cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA256 uint16 = 0x006D + // Unassigned uint16 = 0x006E-83 + cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0084 + cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0085 + cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0086 + cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0087 + cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0088 + cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0089 + cipher_TLS_PSK_WITH_RC4_128_SHA uint16 = 0x008A + cipher_TLS_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0x008B + cipher_TLS_PSK_WITH_AES_128_CBC_SHA uint16 = 0x008C + cipher_TLS_PSK_WITH_AES_256_CBC_SHA uint16 = 0x008D + cipher_TLS_DHE_PSK_WITH_RC4_128_SHA uint16 = 0x008E + cipher_TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0x008F + cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA uint16 = 0x0090 + cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA uint16 = 0x0091 + cipher_TLS_RSA_PSK_WITH_RC4_128_SHA uint16 = 0x0092 + cipher_TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0x0093 + cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA uint16 = 0x0094 + cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA uint16 = 0x0095 + cipher_TLS_RSA_WITH_SEED_CBC_SHA uint16 = 0x0096 + cipher_TLS_DH_DSS_WITH_SEED_CBC_SHA uint16 = 0x0097 + cipher_TLS_DH_RSA_WITH_SEED_CBC_SHA uint16 = 0x0098 + cipher_TLS_DHE_DSS_WITH_SEED_CBC_SHA uint16 = 0x0099 + cipher_TLS_DHE_RSA_WITH_SEED_CBC_SHA uint16 = 0x009A + cipher_TLS_DH_anon_WITH_SEED_CBC_SHA uint16 = 0x009B + cipher_TLS_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0x009C + cipher_TLS_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0x009D + cipher_TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0x009E + cipher_TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0x009F + cipher_TLS_DH_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0x00A0 + cipher_TLS_DH_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0x00A1 + cipher_TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 uint16 = 0x00A2 + cipher_TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 uint16 = 0x00A3 + cipher_TLS_DH_DSS_WITH_AES_128_GCM_SHA256 uint16 = 0x00A4 + cipher_TLS_DH_DSS_WITH_AES_256_GCM_SHA384 uint16 = 0x00A5 + cipher_TLS_DH_anon_WITH_AES_128_GCM_SHA256 uint16 = 0x00A6 + cipher_TLS_DH_anon_WITH_AES_256_GCM_SHA384 uint16 = 0x00A7 + cipher_TLS_PSK_WITH_AES_128_GCM_SHA256 uint16 = 0x00A8 + cipher_TLS_PSK_WITH_AES_256_GCM_SHA384 uint16 = 0x00A9 + cipher_TLS_DHE_PSK_WITH_AES_128_GCM_SHA256 uint16 = 0x00AA + cipher_TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 uint16 = 0x00AB + cipher_TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 uint16 = 0x00AC + cipher_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 uint16 = 0x00AD + cipher_TLS_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0x00AE + cipher_TLS_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0x00AF + cipher_TLS_PSK_WITH_NULL_SHA256 uint16 = 0x00B0 + cipher_TLS_PSK_WITH_NULL_SHA384 uint16 = 0x00B1 + cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0x00B2 + cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0x00B3 + cipher_TLS_DHE_PSK_WITH_NULL_SHA256 uint16 = 0x00B4 + cipher_TLS_DHE_PSK_WITH_NULL_SHA384 uint16 = 0x00B5 + cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0x00B6 + cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0x00B7 + cipher_TLS_RSA_PSK_WITH_NULL_SHA256 uint16 = 0x00B8 + cipher_TLS_RSA_PSK_WITH_NULL_SHA384 uint16 = 0x00B9 + cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BA + cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BB + cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BC + cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BD + cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BE + cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BF + cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C0 + cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C1 + cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C2 + cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C3 + cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C4 + cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C5 + // Unassigned uint16 = 0x00C6-FE + cipher_TLS_EMPTY_RENEGOTIATION_INFO_SCSV uint16 = 0x00FF + // Unassigned uint16 = 0x01-55,* + cipher_TLS_FALLBACK_SCSV uint16 = 0x5600 + // Unassigned uint16 = 0x5601 - 0xC000 + cipher_TLS_ECDH_ECDSA_WITH_NULL_SHA uint16 = 0xC001 + cipher_TLS_ECDH_ECDSA_WITH_RC4_128_SHA uint16 = 0xC002 + cipher_TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC003 + cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA uint16 = 0xC004 + cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA uint16 = 0xC005 + cipher_TLS_ECDHE_ECDSA_WITH_NULL_SHA uint16 = 0xC006 + cipher_TLS_ECDHE_ECDSA_WITH_RC4_128_SHA uint16 = 0xC007 + cipher_TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC008 + cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA uint16 = 0xC009 + cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA uint16 = 0xC00A + cipher_TLS_ECDH_RSA_WITH_NULL_SHA uint16 = 0xC00B + cipher_TLS_ECDH_RSA_WITH_RC4_128_SHA uint16 = 0xC00C + cipher_TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC00D + cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA uint16 = 0xC00E + cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA uint16 = 0xC00F + cipher_TLS_ECDHE_RSA_WITH_NULL_SHA uint16 = 0xC010 + cipher_TLS_ECDHE_RSA_WITH_RC4_128_SHA uint16 = 0xC011 + cipher_TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC012 + cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA uint16 = 0xC013 + cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA uint16 = 0xC014 + cipher_TLS_ECDH_anon_WITH_NULL_SHA uint16 = 0xC015 + cipher_TLS_ECDH_anon_WITH_RC4_128_SHA uint16 = 0xC016 + cipher_TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA uint16 = 0xC017 + cipher_TLS_ECDH_anon_WITH_AES_128_CBC_SHA uint16 = 0xC018 + cipher_TLS_ECDH_anon_WITH_AES_256_CBC_SHA uint16 = 0xC019 + cipher_TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC01A + cipher_TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC01B + cipher_TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA uint16 = 0xC01C + cipher_TLS_SRP_SHA_WITH_AES_128_CBC_SHA uint16 = 0xC01D + cipher_TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA uint16 = 0xC01E + cipher_TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA uint16 = 0xC01F + cipher_TLS_SRP_SHA_WITH_AES_256_CBC_SHA uint16 = 0xC020 + cipher_TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA uint16 = 0xC021 + cipher_TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA uint16 = 0xC022 + cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC023 + cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC024 + cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC025 + cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC026 + cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC027 + cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC028 + cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC029 + cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC02A + cipher_TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC02B + cipher_TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC02C + cipher_TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC02D + cipher_TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC02E + cipher_TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC02F + cipher_TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC030 + cipher_TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC031 + cipher_TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC032 + cipher_TLS_ECDHE_PSK_WITH_RC4_128_SHA uint16 = 0xC033 + cipher_TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0xC034 + cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA uint16 = 0xC035 + cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA uint16 = 0xC036 + cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0xC037 + cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0xC038 + cipher_TLS_ECDHE_PSK_WITH_NULL_SHA uint16 = 0xC039 + cipher_TLS_ECDHE_PSK_WITH_NULL_SHA256 uint16 = 0xC03A + cipher_TLS_ECDHE_PSK_WITH_NULL_SHA384 uint16 = 0xC03B + cipher_TLS_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC03C + cipher_TLS_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC03D + cipher_TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC03E + cipher_TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC03F + cipher_TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC040 + cipher_TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC041 + cipher_TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC042 + cipher_TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC043 + cipher_TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC044 + cipher_TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC045 + cipher_TLS_DH_anon_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC046 + cipher_TLS_DH_anon_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC047 + cipher_TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC048 + cipher_TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC049 + cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC04A + cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC04B + cipher_TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC04C + cipher_TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC04D + cipher_TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC04E + cipher_TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC04F + cipher_TLS_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC050 + cipher_TLS_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC051 + cipher_TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC052 + cipher_TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC053 + cipher_TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC054 + cipher_TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC055 + cipher_TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC056 + cipher_TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC057 + cipher_TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC058 + cipher_TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC059 + cipher_TLS_DH_anon_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC05A + cipher_TLS_DH_anon_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC05B + cipher_TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC05C + cipher_TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC05D + cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC05E + cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC05F + cipher_TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC060 + cipher_TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC061 + cipher_TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC062 + cipher_TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC063 + cipher_TLS_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC064 + cipher_TLS_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC065 + cipher_TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC066 + cipher_TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC067 + cipher_TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC068 + cipher_TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC069 + cipher_TLS_PSK_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC06A + cipher_TLS_PSK_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC06B + cipher_TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC06C + cipher_TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC06D + cipher_TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC06E + cipher_TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC06F + cipher_TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC070 + cipher_TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC071 + cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC072 + cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC073 + cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC074 + cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC075 + cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC076 + cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC077 + cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC078 + cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC079 + cipher_TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC07A + cipher_TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC07B + cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC07C + cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC07D + cipher_TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC07E + cipher_TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC07F + cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC080 + cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC081 + cipher_TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC082 + cipher_TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC083 + cipher_TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC084 + cipher_TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC085 + cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC086 + cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC087 + cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC088 + cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC089 + cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC08A + cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC08B + cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC08C + cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC08D + cipher_TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC08E + cipher_TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC08F + cipher_TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC090 + cipher_TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC091 + cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC092 + cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC093 + cipher_TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC094 + cipher_TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC095 + cipher_TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC096 + cipher_TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC097 + cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC098 + cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC099 + cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC09A + cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC09B + cipher_TLS_RSA_WITH_AES_128_CCM uint16 = 0xC09C + cipher_TLS_RSA_WITH_AES_256_CCM uint16 = 0xC09D + cipher_TLS_DHE_RSA_WITH_AES_128_CCM uint16 = 0xC09E + cipher_TLS_DHE_RSA_WITH_AES_256_CCM uint16 = 0xC09F + cipher_TLS_RSA_WITH_AES_128_CCM_8 uint16 = 0xC0A0 + cipher_TLS_RSA_WITH_AES_256_CCM_8 uint16 = 0xC0A1 + cipher_TLS_DHE_RSA_WITH_AES_128_CCM_8 uint16 = 0xC0A2 + cipher_TLS_DHE_RSA_WITH_AES_256_CCM_8 uint16 = 0xC0A3 + cipher_TLS_PSK_WITH_AES_128_CCM uint16 = 0xC0A4 + cipher_TLS_PSK_WITH_AES_256_CCM uint16 = 0xC0A5 + cipher_TLS_DHE_PSK_WITH_AES_128_CCM uint16 = 0xC0A6 + cipher_TLS_DHE_PSK_WITH_AES_256_CCM uint16 = 0xC0A7 + cipher_TLS_PSK_WITH_AES_128_CCM_8 uint16 = 0xC0A8 + cipher_TLS_PSK_WITH_AES_256_CCM_8 uint16 = 0xC0A9 + cipher_TLS_PSK_DHE_WITH_AES_128_CCM_8 uint16 = 0xC0AA + cipher_TLS_PSK_DHE_WITH_AES_256_CCM_8 uint16 = 0xC0AB + cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CCM uint16 = 0xC0AC + cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CCM uint16 = 0xC0AD + cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 uint16 = 0xC0AE + cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8 uint16 = 0xC0AF + // Unassigned uint16 = 0xC0B0-FF + // Unassigned uint16 = 0xC1-CB,* + // Unassigned uint16 = 0xCC00-A7 + cipher_TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCA8 + cipher_TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCA9 + cipher_TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAA + cipher_TLS_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAB + cipher_TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAC + cipher_TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAD + cipher_TLS_RSA_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAE +) + +// isBadCipher reports whether the cipher is blacklisted by the HTTP/2 spec. +// References: +// https://tools.ietf.org/html/rfc7540#appendix-A +// Reject cipher suites from Appendix A. +// "This list includes those cipher suites that do not +// offer an ephemeral key exchange and those that are +// based on the TLS null, stream or block cipher type" +func isBadCipher(cipher uint16) bool { + switch cipher { + case cipher_TLS_NULL_WITH_NULL_NULL, + cipher_TLS_RSA_WITH_NULL_MD5, + cipher_TLS_RSA_WITH_NULL_SHA, + cipher_TLS_RSA_EXPORT_WITH_RC4_40_MD5, + cipher_TLS_RSA_WITH_RC4_128_MD5, + cipher_TLS_RSA_WITH_RC4_128_SHA, + cipher_TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5, + cipher_TLS_RSA_WITH_IDEA_CBC_SHA, + cipher_TLS_RSA_EXPORT_WITH_DES40_CBC_SHA, + cipher_TLS_RSA_WITH_DES_CBC_SHA, + cipher_TLS_RSA_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA, + cipher_TLS_DH_DSS_WITH_DES_CBC_SHA, + cipher_TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA, + cipher_TLS_DH_RSA_WITH_DES_CBC_SHA, + cipher_TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA, + cipher_TLS_DHE_DSS_WITH_DES_CBC_SHA, + cipher_TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA, + cipher_TLS_DHE_RSA_WITH_DES_CBC_SHA, + cipher_TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_DH_anon_EXPORT_WITH_RC4_40_MD5, + cipher_TLS_DH_anon_WITH_RC4_128_MD5, + cipher_TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA, + cipher_TLS_DH_anon_WITH_DES_CBC_SHA, + cipher_TLS_DH_anon_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_KRB5_WITH_DES_CBC_SHA, + cipher_TLS_KRB5_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_KRB5_WITH_RC4_128_SHA, + cipher_TLS_KRB5_WITH_IDEA_CBC_SHA, + cipher_TLS_KRB5_WITH_DES_CBC_MD5, + cipher_TLS_KRB5_WITH_3DES_EDE_CBC_MD5, + cipher_TLS_KRB5_WITH_RC4_128_MD5, + cipher_TLS_KRB5_WITH_IDEA_CBC_MD5, + cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA, + cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA, + cipher_TLS_KRB5_EXPORT_WITH_RC4_40_SHA, + cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5, + cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5, + cipher_TLS_KRB5_EXPORT_WITH_RC4_40_MD5, + cipher_TLS_PSK_WITH_NULL_SHA, + cipher_TLS_DHE_PSK_WITH_NULL_SHA, + cipher_TLS_RSA_PSK_WITH_NULL_SHA, + cipher_TLS_RSA_WITH_AES_128_CBC_SHA, + cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA, + cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA, + cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA, + cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA, + cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA, + cipher_TLS_RSA_WITH_AES_256_CBC_SHA, + cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA, + cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA, + cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA, + cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA, + cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA, + cipher_TLS_RSA_WITH_NULL_SHA256, + cipher_TLS_RSA_WITH_AES_128_CBC_SHA256, + cipher_TLS_RSA_WITH_AES_256_CBC_SHA256, + cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA256, + cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA256, + cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, + cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA, + cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA, + cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA, + cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA, + cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA, + cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA, + cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, + cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA256, + cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA256, + cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, + cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, + cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA256, + cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA256, + cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA, + cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA, + cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA, + cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA, + cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA, + cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA, + cipher_TLS_PSK_WITH_RC4_128_SHA, + cipher_TLS_PSK_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_PSK_WITH_AES_128_CBC_SHA, + cipher_TLS_PSK_WITH_AES_256_CBC_SHA, + cipher_TLS_DHE_PSK_WITH_RC4_128_SHA, + cipher_TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA, + cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA, + cipher_TLS_RSA_PSK_WITH_RC4_128_SHA, + cipher_TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA, + cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA, + cipher_TLS_RSA_WITH_SEED_CBC_SHA, + cipher_TLS_DH_DSS_WITH_SEED_CBC_SHA, + cipher_TLS_DH_RSA_WITH_SEED_CBC_SHA, + cipher_TLS_DHE_DSS_WITH_SEED_CBC_SHA, + cipher_TLS_DHE_RSA_WITH_SEED_CBC_SHA, + cipher_TLS_DH_anon_WITH_SEED_CBC_SHA, + cipher_TLS_RSA_WITH_AES_128_GCM_SHA256, + cipher_TLS_RSA_WITH_AES_256_GCM_SHA384, + cipher_TLS_DH_RSA_WITH_AES_128_GCM_SHA256, + cipher_TLS_DH_RSA_WITH_AES_256_GCM_SHA384, + cipher_TLS_DH_DSS_WITH_AES_128_GCM_SHA256, + cipher_TLS_DH_DSS_WITH_AES_256_GCM_SHA384, + cipher_TLS_DH_anon_WITH_AES_128_GCM_SHA256, + cipher_TLS_DH_anon_WITH_AES_256_GCM_SHA384, + cipher_TLS_PSK_WITH_AES_128_GCM_SHA256, + cipher_TLS_PSK_WITH_AES_256_GCM_SHA384, + cipher_TLS_RSA_PSK_WITH_AES_128_GCM_SHA256, + cipher_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384, + cipher_TLS_PSK_WITH_AES_128_CBC_SHA256, + cipher_TLS_PSK_WITH_AES_256_CBC_SHA384, + cipher_TLS_PSK_WITH_NULL_SHA256, + cipher_TLS_PSK_WITH_NULL_SHA384, + cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA256, + cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA384, + cipher_TLS_DHE_PSK_WITH_NULL_SHA256, + cipher_TLS_DHE_PSK_WITH_NULL_SHA384, + cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA256, + cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA384, + cipher_TLS_RSA_PSK_WITH_NULL_SHA256, + cipher_TLS_RSA_PSK_WITH_NULL_SHA384, + cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256, + cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256, + cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256, + cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256, + cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256, + cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256, + cipher_TLS_EMPTY_RENEGOTIATION_INFO_SCSV, + cipher_TLS_ECDH_ECDSA_WITH_NULL_SHA, + cipher_TLS_ECDH_ECDSA_WITH_RC4_128_SHA, + cipher_TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA, + cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA, + cipher_TLS_ECDHE_ECDSA_WITH_NULL_SHA, + cipher_TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, + cipher_TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + cipher_TLS_ECDH_RSA_WITH_NULL_SHA, + cipher_TLS_ECDH_RSA_WITH_RC4_128_SHA, + cipher_TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA, + cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA, + cipher_TLS_ECDHE_RSA_WITH_NULL_SHA, + cipher_TLS_ECDHE_RSA_WITH_RC4_128_SHA, + cipher_TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + cipher_TLS_ECDH_anon_WITH_NULL_SHA, + cipher_TLS_ECDH_anon_WITH_RC4_128_SHA, + cipher_TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_ECDH_anon_WITH_AES_128_CBC_SHA, + cipher_TLS_ECDH_anon_WITH_AES_256_CBC_SHA, + cipher_TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_SRP_SHA_WITH_AES_128_CBC_SHA, + cipher_TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA, + cipher_TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA, + cipher_TLS_SRP_SHA_WITH_AES_256_CBC_SHA, + cipher_TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA, + cipher_TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA, + cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, + cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, + cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256, + cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384, + cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, + cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, + cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256, + cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384, + cipher_TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256, + cipher_TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384, + cipher_TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256, + cipher_TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384, + cipher_TLS_ECDHE_PSK_WITH_RC4_128_SHA, + cipher_TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA, + cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA, + cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA, + cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256, + cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384, + cipher_TLS_ECDHE_PSK_WITH_NULL_SHA, + cipher_TLS_ECDHE_PSK_WITH_NULL_SHA256, + cipher_TLS_ECDHE_PSK_WITH_NULL_SHA384, + cipher_TLS_RSA_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_RSA_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_DH_anon_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_DH_anon_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_RSA_WITH_ARIA_128_GCM_SHA256, + cipher_TLS_RSA_WITH_ARIA_256_GCM_SHA384, + cipher_TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256, + cipher_TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384, + cipher_TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256, + cipher_TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384, + cipher_TLS_DH_anon_WITH_ARIA_128_GCM_SHA256, + cipher_TLS_DH_anon_WITH_ARIA_256_GCM_SHA384, + cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256, + cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384, + cipher_TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256, + cipher_TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384, + cipher_TLS_PSK_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_PSK_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_PSK_WITH_ARIA_128_GCM_SHA256, + cipher_TLS_PSK_WITH_ARIA_256_GCM_SHA384, + cipher_TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256, + cipher_TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384, + cipher_TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256, + cipher_TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384, + cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384, + cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384, + cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384, + cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384, + cipher_TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256, + cipher_TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384, + cipher_TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256, + cipher_TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384, + cipher_TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256, + cipher_TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384, + cipher_TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256, + cipher_TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384, + cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256, + cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384, + cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256, + cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384, + cipher_TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256, + cipher_TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384, + cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256, + cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384, + cipher_TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384, + cipher_TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, + cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384, + cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, + cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, + cipher_TLS_RSA_WITH_AES_128_CCM, + cipher_TLS_RSA_WITH_AES_256_CCM, + cipher_TLS_RSA_WITH_AES_128_CCM_8, + cipher_TLS_RSA_WITH_AES_256_CCM_8, + cipher_TLS_PSK_WITH_AES_128_CCM, + cipher_TLS_PSK_WITH_AES_256_CCM, + cipher_TLS_PSK_WITH_AES_128_CCM_8, + cipher_TLS_PSK_WITH_AES_256_CCM_8: + return true + default: + return false + } +} diff --git a/vendor/golang.org/x/net/http2/configure_transport.go b/vendor/golang.org/x/net/http2/configure_transport.go index 4f720f530b0..088d6e2bdb3 100644 --- a/vendor/golang.org/x/net/http2/configure_transport.go +++ b/vendor/golang.org/x/net/http2/configure_transport.go @@ -56,7 +56,7 @@ func configureTransport(t1 *http.Transport) (*Transport, error) { } // registerHTTPSProtocol calls Transport.RegisterProtocol but -// convering panics into errors. +// converting panics into errors. func registerHTTPSProtocol(t *http.Transport, rt http.RoundTripper) (err error) { defer func() { if e := recover(); e != nil { @@ -73,7 +73,7 @@ type noDialH2RoundTripper struct{ t *Transport } func (rt noDialH2RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { res, err := rt.t.RoundTrip(req) - if err == ErrNoCachedConn { + if isNoCachedConnError(err) { return nil, http.ErrSkipAltProtocol } return res, err diff --git a/vendor/golang.org/x/net/http2/databuffer.go b/vendor/golang.org/x/net/http2/databuffer.go new file mode 100644 index 00000000000..a3067f8de74 --- /dev/null +++ b/vendor/golang.org/x/net/http2/databuffer.go @@ -0,0 +1,146 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http2 + +import ( + "errors" + "fmt" + "sync" +) + +// Buffer chunks are allocated from a pool to reduce pressure on GC. +// The maximum wasted space per dataBuffer is 2x the largest size class, +// which happens when the dataBuffer has multiple chunks and there is +// one unread byte in both the first and last chunks. We use a few size +// classes to minimize overheads for servers that typically receive very +// small request bodies. +// +// TODO: Benchmark to determine if the pools are necessary. The GC may have +// improved enough that we can instead allocate chunks like this: +// make([]byte, max(16<<10, expectedBytesRemaining)) +var ( + dataChunkSizeClasses = []int{ + 1 << 10, + 2 << 10, + 4 << 10, + 8 << 10, + 16 << 10, + } + dataChunkPools = [...]sync.Pool{ + {New: func() interface{} { return make([]byte, 1<<10) }}, + {New: func() interface{} { return make([]byte, 2<<10) }}, + {New: func() interface{} { return make([]byte, 4<<10) }}, + {New: func() interface{} { return make([]byte, 8<<10) }}, + {New: func() interface{} { return make([]byte, 16<<10) }}, + } +) + +func getDataBufferChunk(size int64) []byte { + i := 0 + for ; i < len(dataChunkSizeClasses)-1; i++ { + if size <= int64(dataChunkSizeClasses[i]) { + break + } + } + return dataChunkPools[i].Get().([]byte) +} + +func putDataBufferChunk(p []byte) { + for i, n := range dataChunkSizeClasses { + if len(p) == n { + dataChunkPools[i].Put(p) + return + } + } + panic(fmt.Sprintf("unexpected buffer len=%v", len(p))) +} + +// dataBuffer is an io.ReadWriter backed by a list of data chunks. +// Each dataBuffer is used to read DATA frames on a single stream. +// The buffer is divided into chunks so the server can limit the +// total memory used by a single connection without limiting the +// request body size on any single stream. +type dataBuffer struct { + chunks [][]byte + r int // next byte to read is chunks[0][r] + w int // next byte to write is chunks[len(chunks)-1][w] + size int // total buffered bytes + expected int64 // we expect at least this many bytes in future Write calls (ignored if <= 0) +} + +var errReadEmpty = errors.New("read from empty dataBuffer") + +// Read copies bytes from the buffer into p. +// It is an error to read when no data is available. +func (b *dataBuffer) Read(p []byte) (int, error) { + if b.size == 0 { + return 0, errReadEmpty + } + var ntotal int + for len(p) > 0 && b.size > 0 { + readFrom := b.bytesFromFirstChunk() + n := copy(p, readFrom) + p = p[n:] + ntotal += n + b.r += n + b.size -= n + // If the first chunk has been consumed, advance to the next chunk. + if b.r == len(b.chunks[0]) { + putDataBufferChunk(b.chunks[0]) + end := len(b.chunks) - 1 + copy(b.chunks[:end], b.chunks[1:]) + b.chunks[end] = nil + b.chunks = b.chunks[:end] + b.r = 0 + } + } + return ntotal, nil +} + +func (b *dataBuffer) bytesFromFirstChunk() []byte { + if len(b.chunks) == 1 { + return b.chunks[0][b.r:b.w] + } + return b.chunks[0][b.r:] +} + +// Len returns the number of bytes of the unread portion of the buffer. +func (b *dataBuffer) Len() int { + return b.size +} + +// Write appends p to the buffer. +func (b *dataBuffer) Write(p []byte) (int, error) { + ntotal := len(p) + for len(p) > 0 { + // If the last chunk is empty, allocate a new chunk. Try to allocate + // enough to fully copy p plus any additional bytes we expect to + // receive. However, this may allocate less than len(p). + want := int64(len(p)) + if b.expected > want { + want = b.expected + } + chunk := b.lastChunkOrAlloc(want) + n := copy(chunk[b.w:], p) + p = p[n:] + b.w += n + b.size += n + b.expected -= int64(n) + } + return ntotal, nil +} + +func (b *dataBuffer) lastChunkOrAlloc(want int64) []byte { + if len(b.chunks) != 0 { + last := b.chunks[len(b.chunks)-1] + if b.w < len(last) { + return last + } + } + chunk := getDataBufferChunk(want) + b.chunks = append(b.chunks, chunk) + b.w = 0 + return chunk +} diff --git a/vendor/golang.org/x/net/http2/errors.go b/vendor/golang.org/x/net/http2/errors.go index 20fd7626a56..71f2c463178 100644 --- a/vendor/golang.org/x/net/http2/errors.go +++ b/vendor/golang.org/x/net/http2/errors.go @@ -87,13 +87,16 @@ type goAwayFlowError struct{} func (goAwayFlowError) Error() string { return "connection exceeded flow control window size" } -// connErrorReason wraps a ConnectionError with an informative error about why it occurs. - +// connError represents an HTTP/2 ConnectionError error code, along +// with a string (for debugging) explaining why. +// // Errors of this type are only returned by the frame parser functions -// and converted into ConnectionError(ErrCodeProtocol). +// and converted into ConnectionError(Code), after stashing away +// the Reason into the Framer's errDetail field, accessible via +// the (*Framer).ErrorDetail method. type connError struct { - Code ErrCode - Reason string + Code ErrCode // the ConnectionError error code + Reason string // additional reason } func (e connError) Error() string { diff --git a/vendor/golang.org/x/net/http2/fixed_buffer.go b/vendor/golang.org/x/net/http2/fixed_buffer.go deleted file mode 100644 index 47da0f0bf71..00000000000 --- a/vendor/golang.org/x/net/http2/fixed_buffer.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package http2 - -import ( - "errors" -) - -// fixedBuffer is an io.ReadWriter backed by a fixed size buffer. -// It never allocates, but moves old data as new data is written. -type fixedBuffer struct { - buf []byte - r, w int -} - -var ( - errReadEmpty = errors.New("read from empty fixedBuffer") - errWriteFull = errors.New("write on full fixedBuffer") -) - -// Read copies bytes from the buffer into p. -// It is an error to read when no data is available. -func (b *fixedBuffer) Read(p []byte) (n int, err error) { - if b.r == b.w { - return 0, errReadEmpty - } - n = copy(p, b.buf[b.r:b.w]) - b.r += n - if b.r == b.w { - b.r = 0 - b.w = 0 - } - return n, nil -} - -// Len returns the number of bytes of the unread portion of the buffer. -func (b *fixedBuffer) Len() int { - return b.w - b.r -} - -// Write copies bytes from p into the buffer. -// It is an error to write more data than the buffer can hold. -func (b *fixedBuffer) Write(p []byte) (n int, err error) { - // Slide existing data to beginning. - if b.r > 0 && len(p) > len(b.buf)-b.w { - copy(b.buf, b.buf[b.r:b.w]) - b.w -= b.r - b.r = 0 - } - - // Write new data. - n = copy(b.buf[b.w:], p) - b.w += n - if n < len(p) { - err = errWriteFull - } - return n, err -} diff --git a/vendor/golang.org/x/net/http2/frame.go b/vendor/golang.org/x/net/http2/frame.go index 957358837c6..3b14890728f 100644 --- a/vendor/golang.org/x/net/http2/frame.go +++ b/vendor/golang.org/x/net/http2/frame.go @@ -122,7 +122,7 @@ var flagName = map[FrameType]map[Flags]string{ // a frameParser parses a frame given its FrameHeader and payload // bytes. The length of payload will always equal fh.Length (which // might be 0). -type frameParser func(fh FrameHeader, payload []byte) (Frame, error) +type frameParser func(fc *frameCache, fh FrameHeader, payload []byte) (Frame, error) var frameParsers = map[FrameType]frameParser{ FrameData: parseDataFrame, @@ -323,6 +323,8 @@ type Framer struct { debugFramerBuf *bytes.Buffer debugReadLoggerf func(string, ...interface{}) debugWriteLoggerf func(string, ...interface{}) + + frameCache *frameCache // nil if frames aren't reused (default) } func (fr *Framer) maxHeaderListSize() uint32 { @@ -398,6 +400,27 @@ const ( maxFrameSize = 1<<24 - 1 ) +// SetReuseFrames allows the Framer to reuse Frames. +// If called on a Framer, Frames returned by calls to ReadFrame are only +// valid until the next call to ReadFrame. +func (fr *Framer) SetReuseFrames() { + if fr.frameCache != nil { + return + } + fr.frameCache = &frameCache{} +} + +type frameCache struct { + dataFrame DataFrame +} + +func (fc *frameCache) getDataFrame() *DataFrame { + if fc == nil { + return &DataFrame{} + } + return &fc.dataFrame +} + // NewFramer returns a Framer that writes frames to w and reads them from r. func NewFramer(w io.Writer, r io.Reader) *Framer { fr := &Framer{ @@ -477,7 +500,7 @@ func (fr *Framer) ReadFrame() (Frame, error) { if _, err := io.ReadFull(fr.r, payload); err != nil { return nil, err } - f, err := typeFrameParser(fh.Type)(fh, payload) + f, err := typeFrameParser(fh.Type)(fr.frameCache, fh, payload) if err != nil { if ce, ok := err.(connError); ok { return nil, fr.connError(ce.Code, ce.Reason) @@ -565,7 +588,7 @@ func (f *DataFrame) Data() []byte { return f.data } -func parseDataFrame(fh FrameHeader, payload []byte) (Frame, error) { +func parseDataFrame(fc *frameCache, fh FrameHeader, payload []byte) (Frame, error) { if fh.StreamID == 0 { // DATA frames MUST be associated with a stream. If a // DATA frame is received whose stream identifier @@ -574,9 +597,9 @@ func parseDataFrame(fh FrameHeader, payload []byte) (Frame, error) { // PROTOCOL_ERROR. return nil, connError{ErrCodeProtocol, "DATA frame with stream ID 0"} } - f := &DataFrame{ - FrameHeader: fh, - } + f := fc.getDataFrame() + f.FrameHeader = fh + var padSize byte if fh.Flags.Has(FlagDataPadded) { var err error @@ -672,7 +695,7 @@ type SettingsFrame struct { p []byte } -func parseSettingsFrame(fh FrameHeader, p []byte) (Frame, error) { +func parseSettingsFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) { if fh.Flags.Has(FlagSettingsAck) && fh.Length > 0 { // When this (ACK 0x1) bit is set, the payload of the // SETTINGS frame MUST be empty. Receipt of a @@ -774,7 +797,7 @@ type PingFrame struct { func (f *PingFrame) IsAck() bool { return f.Flags.Has(FlagPingAck) } -func parsePingFrame(fh FrameHeader, payload []byte) (Frame, error) { +func parsePingFrame(_ *frameCache, fh FrameHeader, payload []byte) (Frame, error) { if len(payload) != 8 { return nil, ConnectionError(ErrCodeFrameSize) } @@ -814,7 +837,7 @@ func (f *GoAwayFrame) DebugData() []byte { return f.debugData } -func parseGoAwayFrame(fh FrameHeader, p []byte) (Frame, error) { +func parseGoAwayFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) { if fh.StreamID != 0 { return nil, ConnectionError(ErrCodeProtocol) } @@ -854,7 +877,7 @@ func (f *UnknownFrame) Payload() []byte { return f.p } -func parseUnknownFrame(fh FrameHeader, p []byte) (Frame, error) { +func parseUnknownFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) { return &UnknownFrame{fh, p}, nil } @@ -865,7 +888,7 @@ type WindowUpdateFrame struct { Increment uint32 // never read with high bit set } -func parseWindowUpdateFrame(fh FrameHeader, p []byte) (Frame, error) { +func parseWindowUpdateFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) { if len(p) != 4 { return nil, ConnectionError(ErrCodeFrameSize) } @@ -930,7 +953,7 @@ func (f *HeadersFrame) HasPriority() bool { return f.FrameHeader.Flags.Has(FlagHeadersPriority) } -func parseHeadersFrame(fh FrameHeader, p []byte) (_ Frame, err error) { +func parseHeadersFrame(_ *frameCache, fh FrameHeader, p []byte) (_ Frame, err error) { hf := &HeadersFrame{ FrameHeader: fh, } @@ -1067,7 +1090,7 @@ func (p PriorityParam) IsZero() bool { return p == PriorityParam{} } -func parsePriorityFrame(fh FrameHeader, payload []byte) (Frame, error) { +func parsePriorityFrame(_ *frameCache, fh FrameHeader, payload []byte) (Frame, error) { if fh.StreamID == 0 { return nil, connError{ErrCodeProtocol, "PRIORITY frame with stream ID 0"} } @@ -1114,7 +1137,7 @@ type RSTStreamFrame struct { ErrCode ErrCode } -func parseRSTStreamFrame(fh FrameHeader, p []byte) (Frame, error) { +func parseRSTStreamFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) { if len(p) != 4 { return nil, ConnectionError(ErrCodeFrameSize) } @@ -1144,7 +1167,7 @@ type ContinuationFrame struct { headerFragBuf []byte } -func parseContinuationFrame(fh FrameHeader, p []byte) (Frame, error) { +func parseContinuationFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) { if fh.StreamID == 0 { return nil, connError{ErrCodeProtocol, "CONTINUATION frame with stream ID 0"} } @@ -1194,7 +1217,7 @@ func (f *PushPromiseFrame) HeadersEnded() bool { return f.FrameHeader.Flags.Has(FlagPushPromiseEndHeaders) } -func parsePushPromise(fh FrameHeader, p []byte) (_ Frame, err error) { +func parsePushPromise(_ *frameCache, fh FrameHeader, p []byte) (_ Frame, err error) { pp := &PushPromiseFrame{ FrameHeader: fh, } diff --git a/vendor/golang.org/x/net/http2/go16.go b/vendor/golang.org/x/net/http2/go16.go index 2b72855f552..00b2e9e3cf3 100644 --- a/vendor/golang.org/x/net/http2/go16.go +++ b/vendor/golang.org/x/net/http2/go16.go @@ -7,7 +7,6 @@ package http2 import ( - "crypto/tls" "net/http" "time" ) @@ -15,29 +14,3 @@ import ( func transportExpectContinueTimeout(t1 *http.Transport) time.Duration { return t1.ExpectContinueTimeout } - -// isBadCipher reports whether the cipher is blacklisted by the HTTP/2 spec. -func isBadCipher(cipher uint16) bool { - switch cipher { - case tls.TLS_RSA_WITH_RC4_128_SHA, - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, - tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: - // Reject cipher suites from Appendix A. - // "This list includes those cipher suites that do not - // offer an ephemeral key exchange and those that are - // based on the TLS null, stream or block cipher type" - return true - default: - return false - } -} diff --git a/vendor/golang.org/x/net/http2/go18.go b/vendor/golang.org/x/net/http2/go18.go index 633202c39fd..4f30d228a88 100644 --- a/vendor/golang.org/x/net/http2/go18.go +++ b/vendor/golang.org/x/net/http2/go18.go @@ -12,7 +12,11 @@ import ( "net/http" ) -func cloneTLSConfig(c *tls.Config) *tls.Config { return c.Clone() } +func cloneTLSConfig(c *tls.Config) *tls.Config { + c2 := c.Clone() + c2.GetClientCertificate = c.GetClientCertificate // golang.org/issue/19264 + return c2 +} var _ http.Pusher = (*responseWriter)(nil) @@ -48,3 +52,5 @@ func reqGetBody(req *http.Request) func() (io.ReadCloser, error) { func reqBodyIsNoBody(body io.ReadCloser) bool { return body == http.NoBody } + +func go18httpNoBody() io.ReadCloser { return http.NoBody } // for tests only diff --git a/vendor/golang.org/x/net/http2/go19.go b/vendor/golang.org/x/net/http2/go19.go new file mode 100644 index 00000000000..38124ba56e2 --- /dev/null +++ b/vendor/golang.org/x/net/http2/go19.go @@ -0,0 +1,16 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 + +package http2 + +import ( + "net/http" +) + +func configureServer19(s *http.Server, conf *Server) error { + s.RegisterOnShutdown(conf.state.startGracefulShutdown) + return nil +} diff --git a/vendor/golang.org/x/net/http2/hpack/encode.go b/vendor/golang.org/x/net/http2/hpack/encode.go index 6b3b9f8b4fa..1565cf2702d 100644 --- a/vendor/golang.org/x/net/http2/hpack/encode.go +++ b/vendor/golang.org/x/net/http2/hpack/encode.go @@ -39,6 +39,7 @@ func NewEncoder(w io.Writer) *Encoder { tableSizeUpdate: false, w: w, } + e.dynTab.table.init() e.dynTab.setMaxSize(initialHeaderTableSize) return e } @@ -88,29 +89,17 @@ func (e *Encoder) WriteField(f HeaderField) error { // only name matches, i points to that index and nameValueMatch // becomes false. func (e *Encoder) searchTable(f HeaderField) (i uint64, nameValueMatch bool) { - for idx, hf := range staticTable { - if !constantTimeStringCompare(hf.Name, f.Name) { - continue - } - if i == 0 { - i = uint64(idx + 1) - } - if f.Sensitive { - continue - } - if !constantTimeStringCompare(hf.Value, f.Value) { - continue - } - i = uint64(idx + 1) - nameValueMatch = true - return + i, nameValueMatch = staticTable.search(f) + if nameValueMatch { + return i, true } - j, nameValueMatch := e.dynTab.search(f) + j, nameValueMatch := e.dynTab.table.search(f) if nameValueMatch || (i == 0 && j != 0) { - i = j + uint64(len(staticTable)) + return j + uint64(staticTable.len()), nameValueMatch } - return + + return i, false } // SetMaxDynamicTableSize changes the dynamic header table size to v. @@ -217,7 +206,7 @@ func appendVarInt(dst []byte, n byte, i uint64) []byte { } // appendHpackString appends s, as encoded in "String Literal" -// representation, to dst and returns the the extended buffer. +// representation, to dst and returns the extended buffer. // // s will be encoded in Huffman codes only when it produces strictly // shorter byte string. diff --git a/vendor/golang.org/x/net/http2/hpack/hpack.go b/vendor/golang.org/x/net/http2/hpack/hpack.go index 007bc7f45fe..176644acdac 100644 --- a/vendor/golang.org/x/net/http2/hpack/hpack.go +++ b/vendor/golang.org/x/net/http2/hpack/hpack.go @@ -102,6 +102,7 @@ func NewDecoder(maxDynamicTableSize uint32, emitFunc func(f HeaderField)) *Decod emit: emitFunc, emitEnabled: true, } + d.dynTab.table.init() d.dynTab.allowedMaxSize = maxDynamicTableSize d.dynTab.setMaxSize(maxDynamicTableSize) return d @@ -154,12 +155,9 @@ func (d *Decoder) SetAllowedMaxDynamicTableSize(v uint32) { } type dynamicTable struct { - // ents is the FIFO described at // http://http2.github.io/http2-spec/compression.html#rfc.section.2.3.2 - // The newest (low index) is append at the end, and items are - // evicted from the front. - ents []HeaderField - size uint32 + table headerFieldTable + size uint32 // in bytes maxSize uint32 // current maxSize allowedMaxSize uint32 // maxSize may go up to this, inclusive } @@ -169,95 +167,45 @@ func (dt *dynamicTable) setMaxSize(v uint32) { dt.evict() } -// TODO: change dynamicTable to be a struct with a slice and a size int field, -// per http://http2.github.io/http2-spec/compression.html#rfc.section.4.1: -// -// -// Then make add increment the size. maybe the max size should move from Decoder to -// dynamicTable and add should return an ok bool if there was enough space. -// -// Later we'll need a remove operation on dynamicTable. - func (dt *dynamicTable) add(f HeaderField) { - dt.ents = append(dt.ents, f) + dt.table.addEntry(f) dt.size += f.Size() dt.evict() } -// If we're too big, evict old stuff (front of the slice) +// If we're too big, evict old stuff. func (dt *dynamicTable) evict() { - base := dt.ents // keep base pointer of slice - for dt.size > dt.maxSize { - dt.size -= dt.ents[0].Size() - dt.ents = dt.ents[1:] - } - - // Shift slice contents down if we evicted things. - if len(dt.ents) != len(base) { - copy(base, dt.ents) - dt.ents = base[:len(dt.ents)] + var n int + for dt.size > dt.maxSize && n < dt.table.len() { + dt.size -= dt.table.ents[n].Size() + n++ } -} - -// constantTimeStringCompare compares string a and b in a constant -// time manner. -func constantTimeStringCompare(a, b string) bool { - if len(a) != len(b) { - return false - } - - c := byte(0) - - for i := 0; i < len(a); i++ { - c |= a[i] ^ b[i] - } - - return c == 0 -} - -// Search searches f in the table. The return value i is 0 if there is -// no name match. If there is name match or name/value match, i is the -// index of that entry (1-based). If both name and value match, -// nameValueMatch becomes true. -func (dt *dynamicTable) search(f HeaderField) (i uint64, nameValueMatch bool) { - l := len(dt.ents) - for j := l - 1; j >= 0; j-- { - ent := dt.ents[j] - if !constantTimeStringCompare(ent.Name, f.Name) { - continue - } - if i == 0 { - i = uint64(l - j) - } - if f.Sensitive { - continue - } - if !constantTimeStringCompare(ent.Value, f.Value) { - continue - } - i = uint64(l - j) - nameValueMatch = true - return - } - return + dt.table.evictOldest(n) } func (d *Decoder) maxTableIndex() int { - return len(d.dynTab.ents) + len(staticTable) + // This should never overflow. RFC 7540 Section 6.5.2 limits the size of + // the dynamic table to 2^32 bytes, where each entry will occupy more than + // one byte. Further, the staticTable has a fixed, small length. + return d.dynTab.table.len() + staticTable.len() } func (d *Decoder) at(i uint64) (hf HeaderField, ok bool) { - if i < 1 { + // See Section 2.3.3. + if i == 0 { return } + if i <= uint64(staticTable.len()) { + return staticTable.ents[i-1], true + } if i > uint64(d.maxTableIndex()) { return } - if i <= uint64(len(staticTable)) { - return staticTable[i-1], true - } - dents := d.dynTab.ents - return dents[len(dents)-(int(i)-len(staticTable))], true + // In the dynamic table, newer entries have lower indices. + // However, dt.ents[0] is the oldest entry. Hence, dt.ents is + // the reversed dynamic table. + dt := d.dynTab.table + return dt.ents[dt.len()-(int(i)-staticTable.len())], true } // Decode decodes an entire block. diff --git a/vendor/golang.org/x/net/http2/hpack/tables.go b/vendor/golang.org/x/net/http2/hpack/tables.go index b9283a02330..a66cfbea69d 100644 --- a/vendor/golang.org/x/net/http2/hpack/tables.go +++ b/vendor/golang.org/x/net/http2/hpack/tables.go @@ -4,73 +4,200 @@ package hpack -func pair(name, value string) HeaderField { - return HeaderField{Name: name, Value: value} +import ( + "fmt" +) + +// headerFieldTable implements a list of HeaderFields. +// This is used to implement the static and dynamic tables. +type headerFieldTable struct { + // For static tables, entries are never evicted. + // + // For dynamic tables, entries are evicted from ents[0] and added to the end. + // Each entry has a unique id that starts at one and increments for each + // entry that is added. This unique id is stable across evictions, meaning + // it can be used as a pointer to a specific entry. As in hpack, unique ids + // are 1-based. The unique id for ents[k] is k + evictCount + 1. + // + // Zero is not a valid unique id. + // + // evictCount should not overflow in any remotely practical situation. In + // practice, we will have one dynamic table per HTTP/2 connection. If we + // assume a very powerful server that handles 1M QPS per connection and each + // request adds (then evicts) 100 entries from the table, it would still take + // 2M years for evictCount to overflow. + ents []HeaderField + evictCount uint64 + + // byName maps a HeaderField name to the unique id of the newest entry with + // the same name. See above for a definition of "unique id". + byName map[string]uint64 + + // byNameValue maps a HeaderField name/value pair to the unique id of the newest + // entry with the same name and value. See above for a definition of "unique id". + byNameValue map[pairNameValue]uint64 +} + +type pairNameValue struct { + name, value string +} + +func (t *headerFieldTable) init() { + t.byName = make(map[string]uint64) + t.byNameValue = make(map[pairNameValue]uint64) +} + +// len reports the number of entries in the table. +func (t *headerFieldTable) len() int { + return len(t.ents) +} + +// addEntry adds a new entry. +func (t *headerFieldTable) addEntry(f HeaderField) { + id := uint64(t.len()) + t.evictCount + 1 + t.byName[f.Name] = id + t.byNameValue[pairNameValue{f.Name, f.Value}] = id + t.ents = append(t.ents, f) +} + +// evictOldest evicts the n oldest entries in the table. +func (t *headerFieldTable) evictOldest(n int) { + if n > t.len() { + panic(fmt.Sprintf("evictOldest(%v) on table with %v entries", n, t.len())) + } + for k := 0; k < n; k++ { + f := t.ents[k] + id := t.evictCount + uint64(k) + 1 + if t.byName[f.Name] == id { + delete(t.byName, f.Name) + } + if p := (pairNameValue{f.Name, f.Value}); t.byNameValue[p] == id { + delete(t.byNameValue, p) + } + } + copy(t.ents, t.ents[n:]) + for k := t.len() - n; k < t.len(); k++ { + t.ents[k] = HeaderField{} // so strings can be garbage collected + } + t.ents = t.ents[:t.len()-n] + if t.evictCount+uint64(n) < t.evictCount { + panic("evictCount overflow") + } + t.evictCount += uint64(n) +} + +// search finds f in the table. If there is no match, i is 0. +// If both name and value match, i is the matched index and nameValueMatch +// becomes true. If only name matches, i points to that index and +// nameValueMatch becomes false. +// +// The returned index is a 1-based HPACK index. For dynamic tables, HPACK says +// that index 1 should be the newest entry, but t.ents[0] is the oldest entry, +// meaning t.ents is reversed for dynamic tables. Hence, when t is a dynamic +// table, the return value i actually refers to the entry t.ents[t.len()-i]. +// +// All tables are assumed to be a dynamic tables except for the global +// staticTable pointer. +// +// See Section 2.3.3. +func (t *headerFieldTable) search(f HeaderField) (i uint64, nameValueMatch bool) { + if !f.Sensitive { + if id := t.byNameValue[pairNameValue{f.Name, f.Value}]; id != 0 { + return t.idToIndex(id), true + } + } + if id := t.byName[f.Name]; id != 0 { + return t.idToIndex(id), false + } + return 0, false +} + +// idToIndex converts a unique id to an HPACK index. +// See Section 2.3.3. +func (t *headerFieldTable) idToIndex(id uint64) uint64 { + if id <= t.evictCount { + panic(fmt.Sprintf("id (%v) <= evictCount (%v)", id, t.evictCount)) + } + k := id - t.evictCount - 1 // convert id to an index t.ents[k] + if t != staticTable { + return uint64(t.len()) - k // dynamic table + } + return k + 1 } // http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-07#appendix-B -var staticTable = [...]HeaderField{ - pair(":authority", ""), // index 1 (1-based) - pair(":method", "GET"), - pair(":method", "POST"), - pair(":path", "/"), - pair(":path", "/index.html"), - pair(":scheme", "http"), - pair(":scheme", "https"), - pair(":status", "200"), - pair(":status", "204"), - pair(":status", "206"), - pair(":status", "304"), - pair(":status", "400"), - pair(":status", "404"), - pair(":status", "500"), - pair("accept-charset", ""), - pair("accept-encoding", "gzip, deflate"), - pair("accept-language", ""), - pair("accept-ranges", ""), - pair("accept", ""), - pair("access-control-allow-origin", ""), - pair("age", ""), - pair("allow", ""), - pair("authorization", ""), - pair("cache-control", ""), - pair("content-disposition", ""), - pair("content-encoding", ""), - pair("content-language", ""), - pair("content-length", ""), - pair("content-location", ""), - pair("content-range", ""), - pair("content-type", ""), - pair("cookie", ""), - pair("date", ""), - pair("etag", ""), - pair("expect", ""), - pair("expires", ""), - pair("from", ""), - pair("host", ""), - pair("if-match", ""), - pair("if-modified-since", ""), - pair("if-none-match", ""), - pair("if-range", ""), - pair("if-unmodified-since", ""), - pair("last-modified", ""), - pair("link", ""), - pair("location", ""), - pair("max-forwards", ""), - pair("proxy-authenticate", ""), - pair("proxy-authorization", ""), - pair("range", ""), - pair("referer", ""), - pair("refresh", ""), - pair("retry-after", ""), - pair("server", ""), - pair("set-cookie", ""), - pair("strict-transport-security", ""), - pair("transfer-encoding", ""), - pair("user-agent", ""), - pair("vary", ""), - pair("via", ""), - pair("www-authenticate", ""), +var staticTable = newStaticTable() +var staticTableEntries = [...]HeaderField{ + {Name: ":authority"}, + {Name: ":method", Value: "GET"}, + {Name: ":method", Value: "POST"}, + {Name: ":path", Value: "/"}, + {Name: ":path", Value: "/index.html"}, + {Name: ":scheme", Value: "http"}, + {Name: ":scheme", Value: "https"}, + {Name: ":status", Value: "200"}, + {Name: ":status", Value: "204"}, + {Name: ":status", Value: "206"}, + {Name: ":status", Value: "304"}, + {Name: ":status", Value: "400"}, + {Name: ":status", Value: "404"}, + {Name: ":status", Value: "500"}, + {Name: "accept-charset"}, + {Name: "accept-encoding", Value: "gzip, deflate"}, + {Name: "accept-language"}, + {Name: "accept-ranges"}, + {Name: "accept"}, + {Name: "access-control-allow-origin"}, + {Name: "age"}, + {Name: "allow"}, + {Name: "authorization"}, + {Name: "cache-control"}, + {Name: "content-disposition"}, + {Name: "content-encoding"}, + {Name: "content-language"}, + {Name: "content-length"}, + {Name: "content-location"}, + {Name: "content-range"}, + {Name: "content-type"}, + {Name: "cookie"}, + {Name: "date"}, + {Name: "etag"}, + {Name: "expect"}, + {Name: "expires"}, + {Name: "from"}, + {Name: "host"}, + {Name: "if-match"}, + {Name: "if-modified-since"}, + {Name: "if-none-match"}, + {Name: "if-range"}, + {Name: "if-unmodified-since"}, + {Name: "last-modified"}, + {Name: "link"}, + {Name: "location"}, + {Name: "max-forwards"}, + {Name: "proxy-authenticate"}, + {Name: "proxy-authorization"}, + {Name: "range"}, + {Name: "referer"}, + {Name: "refresh"}, + {Name: "retry-after"}, + {Name: "server"}, + {Name: "set-cookie"}, + {Name: "strict-transport-security"}, + {Name: "transfer-encoding"}, + {Name: "user-agent"}, + {Name: "vary"}, + {Name: "via"}, + {Name: "www-authenticate"}, +} + +func newStaticTable() *headerFieldTable { + t := &headerFieldTable{} + t.init() + for _, e := range staticTableEntries[:] { + t.addEntry(e) + } + return t } var huffmanCodes = [256]uint32{ diff --git a/vendor/golang.org/x/net/http2/http2.go b/vendor/golang.org/x/net/http2/http2.go index b6b0f9ad153..71db28a873a 100644 --- a/vendor/golang.org/x/net/http2/http2.go +++ b/vendor/golang.org/x/net/http2/http2.go @@ -312,7 +312,7 @@ func mustUint31(v int32) uint32 { } // bodyAllowedForStatus reports whether a given response status code -// permits a body. See RFC 2616, section 4.4. +// permits a body. See RFC 7230, section 3.3. func bodyAllowedForStatus(status int) bool { switch { case status >= 100 && status <= 199: @@ -376,12 +376,16 @@ func (s *sorter) SortStrings(ss []string) { // validPseudoPath reports whether v is a valid :path pseudo-header // value. It must be either: // -// *) a non-empty string starting with '/', but not with with "//", +// *) a non-empty string starting with '/' // *) the string '*', for OPTIONS requests. // // For now this is only used a quick check for deciding when to clean // up Opaque URLs before sending requests from the Transport. // See golang.org/issue/16847 +// +// We used to enforce that the path also didn't start with "//", but +// Google's GFE accepts such paths and Chrome sends them, so ignore +// that part of the spec. See golang.org/issue/19103. func validPseudoPath(v string) bool { - return (len(v) > 0 && v[0] == '/' && (len(v) == 1 || v[1] != '/')) || v == "*" + return (len(v) > 0 && v[0] == '/') || v == "*" } diff --git a/vendor/golang.org/x/net/http2/not_go16.go b/vendor/golang.org/x/net/http2/not_go16.go index efd2e1282cf..508cebcc4db 100644 --- a/vendor/golang.org/x/net/http2/not_go16.go +++ b/vendor/golang.org/x/net/http2/not_go16.go @@ -7,7 +7,6 @@ package http2 import ( - "crypto/tls" "net/http" "time" ) @@ -20,27 +19,3 @@ func transportExpectContinueTimeout(t1 *http.Transport) time.Duration { return 0 } - -// isBadCipher reports whether the cipher is blacklisted by the HTTP/2 spec. -func isBadCipher(cipher uint16) bool { - switch cipher { - case tls.TLS_RSA_WITH_RC4_128_SHA, - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, - tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: - // Reject cipher suites from Appendix A. - // "This list includes those cipher suites that do not - // offer an ephemeral key exchange and those that are - // based on the TLS null, stream or block cipher type" - return true - default: - return false - } -} diff --git a/vendor/golang.org/x/net/http2/not_go18.go b/vendor/golang.org/x/net/http2/not_go18.go index efbf83c32c4..6f8d3f86fa8 100644 --- a/vendor/golang.org/x/net/http2/not_go18.go +++ b/vendor/golang.org/x/net/http2/not_go18.go @@ -25,3 +25,5 @@ func reqGetBody(req *http.Request) func() (io.ReadCloser, error) { } func reqBodyIsNoBody(io.ReadCloser) bool { return false } + +func go18httpNoBody() io.ReadCloser { return nil } // for tests only diff --git a/vendor/golang.org/x/net/http2/not_go19.go b/vendor/golang.org/x/net/http2/not_go19.go new file mode 100644 index 00000000000..5ae07726b74 --- /dev/null +++ b/vendor/golang.org/x/net/http2/not_go19.go @@ -0,0 +1,16 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.9 + +package http2 + +import ( + "net/http" +) + +func configureServer19(s *http.Server, conf *Server) error { + // not supported prior to go1.9 + return nil +} diff --git a/vendor/golang.org/x/net/http2/pipe.go b/vendor/golang.org/x/net/http2/pipe.go index 914aaf8a7f2..a6140099cb3 100644 --- a/vendor/golang.org/x/net/http2/pipe.go +++ b/vendor/golang.org/x/net/http2/pipe.go @@ -15,8 +15,8 @@ import ( // underlying buffer is an interface. (io.Pipe is always unbuffered) type pipe struct { mu sync.Mutex - c sync.Cond // c.L lazily initialized to &p.mu - b pipeBuffer + c sync.Cond // c.L lazily initialized to &p.mu + b pipeBuffer // nil when done reading err error // read error once empty. non-nil means closed. breakErr error // immediate read error (caller doesn't see rest of b) donec chan struct{} // closed on error @@ -32,6 +32,9 @@ type pipeBuffer interface { func (p *pipe) Len() int { p.mu.Lock() defer p.mu.Unlock() + if p.b == nil { + return 0 + } return p.b.Len() } @@ -47,7 +50,7 @@ func (p *pipe) Read(d []byte) (n int, err error) { if p.breakErr != nil { return 0, p.breakErr } - if p.b.Len() > 0 { + if p.b != nil && p.b.Len() > 0 { return p.b.Read(d) } if p.err != nil { @@ -55,6 +58,7 @@ func (p *pipe) Read(d []byte) (n int, err error) { p.readFn() // e.g. copy trailers p.readFn = nil // not sticky like p.err } + p.b = nil return 0, p.err } p.c.Wait() @@ -75,6 +79,9 @@ func (p *pipe) Write(d []byte) (n int, err error) { if p.err != nil { return 0, errClosedPipeWrite } + if p.breakErr != nil { + return len(d), nil // discard when there is no reader + } return p.b.Write(d) } @@ -109,6 +116,9 @@ func (p *pipe) closeWithError(dst *error, err error, fn func()) { return } p.readFn = fn + if dst == &p.breakErr { + p.b = nil + } *dst = err p.closeDoneLocked() } diff --git a/vendor/golang.org/x/net/http2/server.go b/vendor/golang.org/x/net/http2/server.go index 3c641a8c243..72f65c8f7b7 100644 --- a/vendor/golang.org/x/net/http2/server.go +++ b/vendor/golang.org/x/net/http2/server.go @@ -46,6 +46,7 @@ import ( "sync" "time" + "golang.org/x/net/http/httpguts" "golang.org/x/net/http2/hpack" ) @@ -110,9 +111,41 @@ type Server struct { // activity for the purposes of IdleTimeout. IdleTimeout time.Duration + // MaxUploadBufferPerConnection is the size of the initial flow + // control window for each connections. The HTTP/2 spec does not + // allow this to be smaller than 65535 or larger than 2^32-1. + // If the value is outside this range, a default value will be + // used instead. + MaxUploadBufferPerConnection int32 + + // MaxUploadBufferPerStream is the size of the initial flow control + // window for each stream. The HTTP/2 spec does not allow this to + // be larger than 2^32-1. If the value is zero or larger than the + // maximum, a default value will be used instead. + MaxUploadBufferPerStream int32 + // NewWriteScheduler constructs a write scheduler for a connection. // If nil, a default scheduler is chosen. NewWriteScheduler func() WriteScheduler + + // Internal state. This is a pointer (rather than embedded directly) + // so that we don't embed a Mutex in this struct, which will make the + // struct non-copyable, which might break some callers. + state *serverInternalState +} + +func (s *Server) initialConnRecvWindowSize() int32 { + if s.MaxUploadBufferPerConnection > initialWindowSize { + return s.MaxUploadBufferPerConnection + } + return 1 << 20 +} + +func (s *Server) initialStreamRecvWindowSize() int32 { + if s.MaxUploadBufferPerStream > 0 { + return s.MaxUploadBufferPerStream + } + return 1 << 20 } func (s *Server) maxReadFrameSize() uint32 { @@ -129,6 +162,40 @@ func (s *Server) maxConcurrentStreams() uint32 { return defaultMaxStreams } +type serverInternalState struct { + mu sync.Mutex + activeConns map[*serverConn]struct{} +} + +func (s *serverInternalState) registerConn(sc *serverConn) { + if s == nil { + return // if the Server was used without calling ConfigureServer + } + s.mu.Lock() + s.activeConns[sc] = struct{}{} + s.mu.Unlock() +} + +func (s *serverInternalState) unregisterConn(sc *serverConn) { + if s == nil { + return // if the Server was used without calling ConfigureServer + } + s.mu.Lock() + delete(s.activeConns, sc) + s.mu.Unlock() +} + +func (s *serverInternalState) startGracefulShutdown() { + if s == nil { + return // if the Server was used without calling ConfigureServer + } + s.mu.Lock() + for sc := range s.activeConns { + sc.startGracefulShutdown() + } + s.mu.Unlock() +} + // ConfigureServer adds HTTP/2 support to a net/http Server. // // The configuration conf may be nil. @@ -141,21 +208,28 @@ func ConfigureServer(s *http.Server, conf *Server) error { if conf == nil { conf = new(Server) } + conf.state = &serverInternalState{activeConns: make(map[*serverConn]struct{})} if err := configureServer18(s, conf); err != nil { return err } + if err := configureServer19(s, conf); err != nil { + return err + } if s.TLSConfig == nil { s.TLSConfig = new(tls.Config) } else if s.TLSConfig.CipherSuites != nil { // If they already provided a CipherSuite list, return // an error if it has a bad order or is missing - // ECDHE_RSA_WITH_AES_128_GCM_SHA256. - const requiredCipher = tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + // ECDHE_RSA_WITH_AES_128_GCM_SHA256 or ECDHE_ECDSA_WITH_AES_128_GCM_SHA256. haveRequired := false sawBad := false for i, cs := range s.TLSConfig.CipherSuites { - if cs == requiredCipher { + switch cs { + case tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + // Alternative MTI cipher to not discourage ECDSA-only servers. + // See http://golang.org/cl/30721 for further information. + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256: haveRequired = true } if isBadCipher(cs) { @@ -165,7 +239,7 @@ func ConfigureServer(s *http.Server, conf *Server) error { } } if !haveRequired { - return fmt.Errorf("http2: TLSConfig.CipherSuites is missing HTTP/2-required TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256") + return fmt.Errorf("http2: TLSConfig.CipherSuites is missing an HTTP/2-required AES_128_GCM_SHA256 cipher.") } } @@ -255,35 +329,37 @@ func (s *Server) ServeConn(c net.Conn, opts *ServeConnOpts) { defer cancel() sc := &serverConn{ - srv: s, - hs: opts.baseConfig(), - conn: c, - baseCtx: baseCtx, - remoteAddrStr: c.RemoteAddr().String(), - bw: newBufferedWriter(c), - handler: opts.handler(), - streams: make(map[uint32]*stream), - readFrameCh: make(chan readFrameResult), - wantWriteFrameCh: make(chan FrameWriteRequest, 8), - wantStartPushCh: make(chan startPushRequest, 8), - wroteFrameCh: make(chan frameWriteResult, 1), // buffered; one send in writeFrameAsync - bodyReadCh: make(chan bodyReadMsg), // buffering doesn't matter either way - doneServing: make(chan struct{}), - clientMaxStreams: math.MaxUint32, // Section 6.5.2: "Initially, there is no limit to this value" - advMaxStreams: s.maxConcurrentStreams(), - initialWindowSize: initialWindowSize, - maxFrameSize: initialMaxFrameSize, - headerTableSize: initialHeaderTableSize, - serveG: newGoroutineLock(), - pushEnabled: true, - } + srv: s, + hs: opts.baseConfig(), + conn: c, + baseCtx: baseCtx, + remoteAddrStr: c.RemoteAddr().String(), + bw: newBufferedWriter(c), + handler: opts.handler(), + streams: make(map[uint32]*stream), + readFrameCh: make(chan readFrameResult), + wantWriteFrameCh: make(chan FrameWriteRequest, 8), + serveMsgCh: make(chan interface{}, 8), + wroteFrameCh: make(chan frameWriteResult, 1), // buffered; one send in writeFrameAsync + bodyReadCh: make(chan bodyReadMsg), // buffering doesn't matter either way + doneServing: make(chan struct{}), + clientMaxStreams: math.MaxUint32, // Section 6.5.2: "Initially, there is no limit to this value" + advMaxStreams: s.maxConcurrentStreams(), + initialStreamSendWindowSize: initialWindowSize, + maxFrameSize: initialMaxFrameSize, + headerTableSize: initialHeaderTableSize, + serveG: newGoroutineLock(), + pushEnabled: true, + } + + s.state.registerConn(sc) + defer s.state.unregisterConn(sc) // The net/http package sets the write deadline from the // http.Server.WriteTimeout during the TLS handshake, but then - // passes the connection off to us with the deadline already - // set. Disarm it here so that it is not applied to additional - // streams opened on this connection. - // TODO: implement WriteTimeout fully. See Issue 18437. + // passes the connection off to us with the deadline already set. + // Write deadlines are set per stream in serverConn.newStream. + // Disarm the net.Conn write deadline here. if sc.hs.WriteTimeout != 0 { sc.conn.SetWriteDeadline(time.Time{}) } @@ -294,6 +370,9 @@ func (s *Server) ServeConn(c net.Conn, opts *ServeConnOpts) { sc.writeSched = NewRandomWriteScheduler() } + // These start at the RFC-specified defaults. If there is a higher + // configured value for inflow, that will be updated when we send a + // WINDOW_UPDATE shortly after sending SETTINGS. sc.flow.add(initialWindowSize) sc.inflow.add(initialWindowSize) sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf) @@ -328,7 +407,7 @@ func (s *Server) ServeConn(c net.Conn, opts *ServeConnOpts) { // addresses during development. // // TODO: optionally enforce? Or enforce at the time we receive - // a new request, and verify the the ServerName matches the :authority? + // a new request, and verify the ServerName matches the :authority? // But that precludes proxy situations, perhaps. // // So for now, do nothing here again. @@ -376,10 +455,9 @@ type serverConn struct { doneServing chan struct{} // closed when serverConn.serve ends readFrameCh chan readFrameResult // written by serverConn.readFrames wantWriteFrameCh chan FrameWriteRequest // from handlers -> serve - wantStartPushCh chan startPushRequest // from handlers -> serve wroteFrameCh chan frameWriteResult // from writeFrameAsync -> serve, tickles more frame writes bodyReadCh chan bodyReadMsg // from handlers -> serve - testHookCh chan func(int) // code to run on the serve loop + serveMsgCh chan interface{} // misc messages & code to send to / run on the serve loop flow flow // conn-wide (not stream-specific) outbound flow control inflow flow // conn-wide inbound flow control tlsState *tls.ConnectionState // shared by all handlers, like net/http @@ -387,38 +465,39 @@ type serverConn struct { writeSched WriteScheduler // Everything following is owned by the serve loop; use serveG.check(): - serveG goroutineLock // used to verify funcs are on serve() - pushEnabled bool - sawFirstSettings bool // got the initial SETTINGS frame after the preface - needToSendSettingsAck bool - unackedSettings int // how many SETTINGS have we sent without ACKs? - clientMaxStreams uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit) - advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client - curClientStreams uint32 // number of open streams initiated by the client - curPushedStreams uint32 // number of open streams initiated by server push - maxClientStreamID uint32 // max ever seen from client (odd), or 0 if there have been no client requests - maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes - streams map[uint32]*stream - initialWindowSize int32 - maxFrameSize int32 - headerTableSize uint32 - peerMaxHeaderListSize uint32 // zero means unknown (default) - canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case - writingFrame bool // started writing a frame (on serve goroutine or separate) - writingFrameAsync bool // started a frame on its own goroutine but haven't heard back on wroteFrameCh - needsFrameFlush bool // last frame write wasn't a flush - inGoAway bool // we've started to or sent GOAWAY - inFrameScheduleLoop bool // whether we're in the scheduleFrameWrite loop - needToSendGoAway bool // we need to schedule a GOAWAY frame write - goAwayCode ErrCode - shutdownTimerCh <-chan time.Time // nil until used - shutdownTimer *time.Timer // nil until used - idleTimer *time.Timer // nil if unused - idleTimerCh <-chan time.Time // nil if unused + serveG goroutineLock // used to verify funcs are on serve() + pushEnabled bool + sawFirstSettings bool // got the initial SETTINGS frame after the preface + needToSendSettingsAck bool + unackedSettings int // how many SETTINGS have we sent without ACKs? + clientMaxStreams uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit) + advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client + curClientStreams uint32 // number of open streams initiated by the client + curPushedStreams uint32 // number of open streams initiated by server push + maxClientStreamID uint32 // max ever seen from client (odd), or 0 if there have been no client requests + maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes + streams map[uint32]*stream + initialStreamSendWindowSize int32 + maxFrameSize int32 + headerTableSize uint32 + peerMaxHeaderListSize uint32 // zero means unknown (default) + canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case + writingFrame bool // started writing a frame (on serve goroutine or separate) + writingFrameAsync bool // started a frame on its own goroutine but haven't heard back on wroteFrameCh + needsFrameFlush bool // last frame write wasn't a flush + inGoAway bool // we've started to or sent GOAWAY + inFrameScheduleLoop bool // whether we're in the scheduleFrameWrite loop + needToSendGoAway bool // we need to schedule a GOAWAY frame write + goAwayCode ErrCode + shutdownTimer *time.Timer // nil until used + idleTimer *time.Timer // nil if unused // Owned by the writeFrameAsync goroutine: headerWriteBuf bytes.Buffer hpackEncoder *hpack.Encoder + + // Used by startGracefulShutdown. + shutdownOnce sync.Once } func (sc *serverConn) maxHeaderListSize() uint32 { @@ -463,10 +542,10 @@ type stream struct { numTrailerValues int64 weight uint8 state streamState - resetQueued bool // RST_STREAM queued for write; set by sc.resetStream - gotTrailerHeader bool // HEADER frame for trailers was seen - wroteHeaders bool // whether we wrote headers (not status 100) - reqBuf []byte // if non-nil, body pipe buffer to return later at EOF + resetQueued bool // RST_STREAM queued for write; set by sc.resetStream + gotTrailerHeader bool // HEADER frame for trailers was seen + wroteHeaders bool // whether we wrote headers (not status 100) + writeDeadline *time.Timer // nil if unused trailer http.Header // accumulated trailers reqTrailer http.Header // handler's Request.Trailer @@ -574,7 +653,7 @@ func (sc *serverConn) condlogf(err error, format string, args ...interface{}) { if err == nil { return } - if err == io.EOF || err == io.ErrUnexpectedEOF || isClosedConnError(err) { + if err == io.EOF || err == io.ErrUnexpectedEOF || isClosedConnError(err) || err == errPrefaceTimeout { // Boring, expected errors. sc.vlogf(format, args...) } else { @@ -696,15 +775,17 @@ func (sc *serverConn) serve() { {SettingMaxFrameSize, sc.srv.maxReadFrameSize()}, {SettingMaxConcurrentStreams, sc.advMaxStreams}, {SettingMaxHeaderListSize, sc.maxHeaderListSize()}, - - // TODO: more actual settings, notably - // SettingInitialWindowSize, but then we also - // want to bump up the conn window size the - // same amount here right after the settings + {SettingInitialWindowSize, uint32(sc.srv.initialStreamRecvWindowSize())}, }, }) sc.unackedSettings++ + // Each connection starts with intialWindowSize inflow tokens. + // If a higher value is configured, we add more tokens. + if diff := sc.srv.initialConnRecvWindowSize() - initialWindowSize; diff > 0 { + sc.sendWindowUpdate(nil, int(diff)) + } + if err := sc.readPreface(); err != nil { sc.condlogf(err, "http2: server: error reading preface from client %v: %v", sc.conn.RemoteAddr(), err) return @@ -717,27 +798,25 @@ func (sc *serverConn) serve() { sc.setConnState(http.StateIdle) if sc.srv.IdleTimeout != 0 { - sc.idleTimer = time.NewTimer(sc.srv.IdleTimeout) + sc.idleTimer = time.AfterFunc(sc.srv.IdleTimeout, sc.onIdleTimer) defer sc.idleTimer.Stop() - sc.idleTimerCh = sc.idleTimer.C - } - - var gracefulShutdownCh <-chan struct{} - if sc.hs != nil { - gracefulShutdownCh = h1ServerShutdownChan(sc.hs) } go sc.readFrames() // closed by defer sc.conn.Close above - settingsTimer := time.NewTimer(firstSettingsTimeout) + settingsTimer := time.AfterFunc(firstSettingsTimeout, sc.onSettingsTimer) + defer settingsTimer.Stop() + loopNum := 0 for { loopNum++ select { case wr := <-sc.wantWriteFrameCh: + if se, ok := wr.write.(StreamError); ok { + sc.resetStream(se) + break + } sc.writeFrame(wr) - case spr := <-sc.wantStartPushCh: - sc.startPush(spr) case res := <-sc.wroteFrameCh: sc.wroteFrame(res) case res := <-sc.readFrameCh: @@ -745,36 +824,85 @@ func (sc *serverConn) serve() { return } res.readMore() - if settingsTimer.C != nil { + if settingsTimer != nil { settingsTimer.Stop() - settingsTimer.C = nil + settingsTimer = nil } case m := <-sc.bodyReadCh: sc.noteBodyRead(m.st, m.n) - case <-settingsTimer.C: - sc.logf("timeout waiting for SETTINGS frames from %v", sc.conn.RemoteAddr()) - return - case <-gracefulShutdownCh: - gracefulShutdownCh = nil - sc.startGracefulShutdown() - case <-sc.shutdownTimerCh: - sc.vlogf("GOAWAY close timer fired; closing conn from %v", sc.conn.RemoteAddr()) - return - case <-sc.idleTimerCh: - sc.vlogf("connection is idle") - sc.goAway(ErrCodeNo) - case fn := <-sc.testHookCh: - fn(loopNum) + case msg := <-sc.serveMsgCh: + switch v := msg.(type) { + case func(int): + v(loopNum) // for testing + case *serverMessage: + switch v { + case settingsTimerMsg: + sc.logf("timeout waiting for SETTINGS frames from %v", sc.conn.RemoteAddr()) + return + case idleTimerMsg: + sc.vlogf("connection is idle") + sc.goAway(ErrCodeNo) + case shutdownTimerMsg: + sc.vlogf("GOAWAY close timer fired; closing conn from %v", sc.conn.RemoteAddr()) + return + case gracefulShutdownMsg: + sc.startGracefulShutdownInternal() + default: + panic("unknown timer") + } + case *startPushRequest: + sc.startPush(v) + default: + panic(fmt.Sprintf("unexpected type %T", v)) + } } - if sc.inGoAway && sc.curOpenStreams() == 0 && !sc.needToSendGoAway && !sc.writingFrame { - return + // Start the shutdown timer after sending a GOAWAY. When sending GOAWAY + // with no error code (graceful shutdown), don't start the timer until + // all open streams have been completed. + sentGoAway := sc.inGoAway && !sc.needToSendGoAway && !sc.writingFrame + gracefulShutdownComplete := sc.goAwayCode == ErrCodeNo && sc.curOpenStreams() == 0 + if sentGoAway && sc.shutdownTimer == nil && (sc.goAwayCode != ErrCodeNo || gracefulShutdownComplete) { + sc.shutDownIn(goAwayTimeout) } } } -// readPreface reads the ClientPreface greeting from the peer -// or returns an error on timeout or an invalid greeting. +func (sc *serverConn) awaitGracefulShutdown(sharedCh <-chan struct{}, privateCh chan struct{}) { + select { + case <-sc.doneServing: + case <-sharedCh: + close(privateCh) + } +} + +type serverMessage int + +// Message values sent to serveMsgCh. +var ( + settingsTimerMsg = new(serverMessage) + idleTimerMsg = new(serverMessage) + shutdownTimerMsg = new(serverMessage) + gracefulShutdownMsg = new(serverMessage) +) + +func (sc *serverConn) onSettingsTimer() { sc.sendServeMsg(settingsTimerMsg) } +func (sc *serverConn) onIdleTimer() { sc.sendServeMsg(idleTimerMsg) } +func (sc *serverConn) onShutdownTimer() { sc.sendServeMsg(shutdownTimerMsg) } + +func (sc *serverConn) sendServeMsg(msg interface{}) { + sc.serveG.checkNotOn() // NOT + select { + case sc.serveMsgCh <- msg: + case <-sc.doneServing: + } +} + +var errPrefaceTimeout = errors.New("timeout waiting for client preface") + +// readPreface reads the ClientPreface greeting from the peer or +// returns errPrefaceTimeout on timeout, or an error if the greeting +// is invalid. func (sc *serverConn) readPreface() error { errc := make(chan error, 1) go func() { @@ -792,7 +920,7 @@ func (sc *serverConn) readPreface() error { defer timer.Stop() select { case <-timer.C: - return errors.New("timeout waiting for client preface") + return errPrefaceTimeout case err := <-errc: if err == nil { if VerboseLogs { @@ -1014,7 +1142,11 @@ func (sc *serverConn) wroteFrame(res frameWriteResult) { // stateClosed after the RST_STREAM frame is // written. st.state = stateHalfClosedLocal - sc.resetStream(streamError(st.id, ErrCodeCancel)) + // Section 8.1: a server MAY request that the client abort + // transmission of a request without error by sending a + // RST_STREAM with an error code of NO_ERROR after sending + // a complete response. + sc.resetStream(streamError(st.id, ErrCodeNo)) case stateHalfClosedRemote: sc.closeStream(st, errHandlerComplete) } @@ -1086,33 +1218,43 @@ func (sc *serverConn) scheduleFrameWrite() { sc.inFrameScheduleLoop = false } -// startGracefulShutdown sends a GOAWAY with ErrCodeNo to tell the -// client we're gracefully shutting down. The connection isn't closed -// until all current streams are done. +// startGracefulShutdown gracefully shuts down a connection. This +// sends GOAWAY with ErrCodeNo to tell the client we're gracefully +// shutting down. The connection isn't closed until all current +// streams are done. +// +// startGracefulShutdown returns immediately; it does not wait until +// the connection has shut down. func (sc *serverConn) startGracefulShutdown() { - sc.goAwayIn(ErrCodeNo, 0) + sc.serveG.checkNotOn() // NOT + sc.shutdownOnce.Do(func() { sc.sendServeMsg(gracefulShutdownMsg) }) } -func (sc *serverConn) goAway(code ErrCode) { - sc.serveG.check() - var forceCloseIn time.Duration - if code != ErrCodeNo { - forceCloseIn = 250 * time.Millisecond - } else { - // TODO: configurable - forceCloseIn = 1 * time.Second - } - sc.goAwayIn(code, forceCloseIn) +// After sending GOAWAY, the connection will close after goAwayTimeout. +// If we close the connection immediately after sending GOAWAY, there may +// be unsent data in our kernel receive buffer, which will cause the kernel +// to send a TCP RST on close() instead of a FIN. This RST will abort the +// connection immediately, whether or not the client had received the GOAWAY. +// +// Ideally we should delay for at least 1 RTT + epsilon so the client has +// a chance to read the GOAWAY and stop sending messages. Measuring RTT +// is hard, so we approximate with 1 second. See golang.org/issue/18701. +// +// This is a var so it can be shorter in tests, where all requests uses the +// loopback interface making the expected RTT very small. +// +// TODO: configurable? +var goAwayTimeout = 1 * time.Second + +func (sc *serverConn) startGracefulShutdownInternal() { + sc.goAway(ErrCodeNo) } -func (sc *serverConn) goAwayIn(code ErrCode, forceCloseIn time.Duration) { +func (sc *serverConn) goAway(code ErrCode) { sc.serveG.check() if sc.inGoAway { return } - if forceCloseIn != 0 { - sc.shutDownIn(forceCloseIn) - } sc.inGoAway = true sc.needToSendGoAway = true sc.goAwayCode = code @@ -1121,8 +1263,7 @@ func (sc *serverConn) goAwayIn(code ErrCode, forceCloseIn time.Duration) { func (sc *serverConn) shutDownIn(d time.Duration) { sc.serveG.check() - sc.shutdownTimer = time.NewTimer(d) - sc.shutdownTimerCh = sc.shutdownTimer.C + sc.shutdownTimer = time.AfterFunc(d, sc.onShutdownTimer) } func (sc *serverConn) resetStream(se StreamError) { @@ -1305,6 +1446,9 @@ func (sc *serverConn) closeStream(st *stream, err error) { panic(fmt.Sprintf("invariant; can't close stream in state %v", st.state)) } st.state = stateClosed + if st.writeDeadline != nil { + st.writeDeadline.Stop() + } if st.isPushed() { sc.curPushedStreams-- } else { @@ -1317,7 +1461,7 @@ func (sc *serverConn) closeStream(st *stream, err error) { sc.idleTimer.Reset(sc.srv.IdleTimeout) } if h1ServerKeepAlivesDisabled(sc.hs) { - sc.startGracefulShutdown() + sc.startGracefulShutdownInternal() } } if p := st.body; p != nil { @@ -1395,9 +1539,9 @@ func (sc *serverConn) processSettingInitialWindowSize(val uint32) error { // adjust the size of all stream flow control windows that it // maintains by the difference between the new value and the // old value." - old := sc.initialWindowSize - sc.initialWindowSize = int32(val) - growth := sc.initialWindowSize - old // may be negative + old := sc.initialStreamSendWindowSize + sc.initialStreamSendWindowSize = int32(val) + growth := int32(val) - old // may be negative for _, st := range sc.streams { if !st.flow.add(growth) { // 6.9.2 Initial Flow Control Window Size @@ -1504,7 +1648,7 @@ func (sc *serverConn) processGoAway(f *GoAwayFrame) error { } else { sc.vlogf("http2: received GOAWAY %+v, starting graceful shutdown", f) } - sc.startGracefulShutdown() + sc.startGracefulShutdownInternal() // http://tools.ietf.org/html/rfc7540#section-6.8 // We should not create any new streams, which means we should disable push. sc.pushEnabled = false @@ -1543,6 +1687,12 @@ func (st *stream) copyTrailersToHandlerRequest() { } } +// onWriteTimeout is run on its own goroutine (from time.AfterFunc) +// when the stream's WriteTimeout has fired. +func (st *stream) onWriteTimeout() { + st.sc.writeFrameFromHandler(FrameWriteRequest{write: streamError(st.id, ErrCodeInternal)}) +} + func (sc *serverConn) processHeaders(f *MetaHeadersFrame) error { sc.serveG.check() id := f.StreamID @@ -1668,7 +1818,7 @@ func (st *stream) processTrailerHeaders(f *MetaHeadersFrame) error { if st.trailer != nil { for _, hf := range f.RegularFields() { key := sc.canonicalHeader(hf.Name) - if !ValidTrailerHeader(key) { + if !httpguts.ValidTrailerHeader(key) { // TODO: send more details to the peer somehow. But http2 has // no way to send debug data at a stream level. Discuss with // HTTP folk. @@ -1719,9 +1869,12 @@ func (sc *serverConn) newStream(id, pusherID uint32, state streamState) *stream } st.cw.Init() st.flow.conn = &sc.flow // link to conn-level counter - st.flow.add(sc.initialWindowSize) - st.inflow.conn = &sc.inflow // link to conn-level counter - st.inflow.add(initialWindowSize) // TODO: update this when we send a higher initial window size in the initial settings + st.flow.add(sc.initialStreamSendWindowSize) + st.inflow.conn = &sc.inflow // link to conn-level counter + st.inflow.add(sc.srv.initialStreamRecvWindowSize()) + if sc.hs.WriteTimeout != 0 { + st.writeDeadline = time.AfterFunc(sc.hs.WriteTimeout, st.onWriteTimeout) + } sc.streams[id] = st sc.writeSched.OpenStream(st.id, OpenStreamOptions{PusherID: pusherID}) @@ -1785,16 +1938,14 @@ func (sc *serverConn) newWriterAndRequest(st *stream, f *MetaHeadersFrame) (*res return nil, nil, err } if bodyOpen { - st.reqBuf = getRequestBodyBuf() - req.Body.(*requestBody).pipe = &pipe{ - b: &fixedBuffer{buf: st.reqBuf}, - } - if vv, ok := rp.header["Content-Length"]; ok { req.ContentLength, _ = strconv.ParseInt(vv[0], 10, 64) } else { req.ContentLength = -1 } + req.Body.(*requestBody).pipe = &pipe{ + b: &dataBuffer{expected: req.ContentLength}, + } } return rw, req, nil } @@ -1890,24 +2041,6 @@ func (sc *serverConn) newWriterAndRequestNoBody(st *stream, rp requestParam) (*r return rw, req, nil } -var reqBodyCache = make(chan []byte, 8) - -func getRequestBodyBuf() []byte { - select { - case b := <-reqBodyCache: - return b - default: - return make([]byte, initialWindowSize) - } -} - -func putRequestBodyBuf(b []byte) { - select { - case reqBodyCache <- b: - default: - } -} - // Run on its own goroutine. func (sc *serverConn) runHandler(rw *responseWriter, req *http.Request, handler func(http.ResponseWriter, *http.Request)) { didPanic := true @@ -2003,12 +2136,6 @@ func (sc *serverConn) noteBodyReadFromHandler(st *stream, n int, err error) { case <-sc.doneServing: } } - if err == io.EOF { - if buf := st.reqBuf; buf != nil { - st.reqBuf = nil // shouldn't matter; field unused by other - putRequestBodyBuf(buf) - } - } } func (sc *serverConn) noteBodyRead(st *stream, n int) { @@ -2138,6 +2265,7 @@ type responseWriterState struct { wroteHeader bool // WriteHeader called (explicitly or implicitly). Not necessarily sent to user yet. sentHeader bool // have we sent the header frame? handlerDone bool // handler has finished + dirty bool // a Write failed; don't reuse this responseWriterState sentContentLen int64 // non-zero if handler set a Content-Length header wroteBytes int64 @@ -2157,8 +2285,8 @@ func (rws *responseWriterState) hasTrailers() bool { return len(rws.trailers) != // written in the trailers at the end of the response. func (rws *responseWriterState) declareTrailer(k string) { k = http.CanonicalHeaderKey(k) - if !ValidTrailerHeader(k) { - // Forbidden by RFC 2616 14.40. + if !httpguts.ValidTrailerHeader(k) { + // Forbidden by RFC 7230, section 4.1.2. rws.conn.logf("ignoring invalid trailer %q", k) return } @@ -2195,8 +2323,16 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { clen = strconv.Itoa(len(p)) } _, hasContentType := rws.snapHeader["Content-Type"] - if !hasContentType && bodyAllowedForStatus(rws.status) { - ctype = http.DetectContentType(p) + if !hasContentType && bodyAllowedForStatus(rws.status) && len(p) > 0 { + if cto := rws.snapHeader.Get("X-Content-Type-Options"); strings.EqualFold("nosniff", cto) { + // nosniff is an explicit directive not to guess a content-type. + // Content-sniffing is no less susceptible to polyglot attacks via + // hosted content when done on the server. + ctype = "application/octet-stream" + rws.conn.logf("http2: WriteHeader called with X-Content-Type-Options:nosniff but no Content-Type") + } else { + ctype = http.DetectContentType(p) + } } var date string if _, ok := rws.snapHeader["Date"]; !ok { @@ -2219,6 +2355,7 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { date: date, }) if err != nil { + rws.dirty = true return 0, err } if endStream { @@ -2240,6 +2377,7 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { if len(p) > 0 || endStream { // only send a 0 byte DATA frame if we're ending the stream. if err := rws.conn.writeDataFromHandler(rws.stream, p, endStream); err != nil { + rws.dirty = true return 0, err } } @@ -2251,6 +2389,9 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { trailers: rws.trailers, endStream: true, }) + if err != nil { + rws.dirty = true + } return len(p), err } return len(p), nil @@ -2274,7 +2415,7 @@ const TrailerPrefix = "Trailer:" // after the header has already been flushed. Because the Go // ResponseWriter interface has no way to set Trailers (only the // Header), and because we didn't want to expand the ResponseWriter -// interface, and because nobody used trailers, and because RFC 2616 +// interface, and because nobody used trailers, and because RFC 7230 // says you SHOULD (but not must) predeclare any trailers in the // header, the official ResponseWriter rules said trailers in Go must // be predeclared, and then we reuse the same ResponseWriter.Header() @@ -2358,6 +2499,24 @@ func (w *responseWriter) Header() http.Header { return rws.handlerHeader } +// checkWriteHeaderCode is a copy of net/http's checkWriteHeaderCode. +func checkWriteHeaderCode(code int) { + // Issue 22880: require valid WriteHeader status codes. + // For now we only enforce that it's three digits. + // In the future we might block things over 599 (600 and above aren't defined + // at http://httpwg.org/specs/rfc7231.html#status.codes) + // and we might block under 200 (once we have more mature 1xx support). + // But for now any three digits. + // + // We used to send "HTTP/1.1 000 0" on the wire in responses but there's + // no equivalent bogus thing we can realistically send in HTTP/2, + // so we'll consistently panic instead and help people find their bugs + // early. (We can't return an error from WriteHeader even if we wanted to.) + if code < 100 || code > 999 { + panic(fmt.Sprintf("invalid WriteHeader code %v", code)) + } +} + func (w *responseWriter) WriteHeader(code int) { rws := w.rws if rws == nil { @@ -2368,6 +2527,7 @@ func (w *responseWriter) WriteHeader(code int) { func (rws *responseWriterState) writeHeader(code int) { if !rws.wroteHeader { + checkWriteHeaderCode(code) rws.wroteHeader = true rws.status = code if len(rws.handlerHeader) > 0 { @@ -2390,7 +2550,7 @@ func cloneHeader(h http.Header) http.Header { // // * Handler calls w.Write or w.WriteString -> // * -> rws.bw (*bufio.Writer) -> -// * (Handler migth call Flush) +// * (Handler might call Flush) // * -> chunkWriter{rws} // * -> responseWriterState.writeChunk(p []byte) // * -> responseWriterState.writeChunk (most of the magic; see comment there) @@ -2429,10 +2589,19 @@ func (w *responseWriter) write(lenData int, dataB []byte, dataS string) (n int, func (w *responseWriter) handlerDone() { rws := w.rws + dirty := rws.dirty rws.handlerDone = true w.Flush() w.rws = nil - responseWriterStatePool.Put(rws) + if !dirty { + // Only recycle the pool if all prior Write calls to + // the serverConn goroutine completed successfully. If + // they returned earlier due to resets from the peer + // there might still be write goroutines outstanding + // from the serverConn referencing the rws memory. See + // issue 20704. + responseWriterStatePool.Put(rws) + } } // Push errors. @@ -2514,7 +2683,7 @@ func (w *responseWriter) push(target string, opts pushOptions) error { return fmt.Errorf("method %q must be GET or HEAD", opts.Method) } - msg := startPushRequest{ + msg := &startPushRequest{ parent: st, method: opts.Method, url: u, @@ -2527,7 +2696,7 @@ func (w *responseWriter) push(target string, opts pushOptions) error { return errClientDisconnected case <-st.cw: return errStreamClosed - case sc.wantStartPushCh <- msg: + case sc.serveMsgCh <- msg: } select { @@ -2549,7 +2718,7 @@ type startPushRequest struct { done chan error } -func (sc *serverConn) startPush(msg startPushRequest) { +func (sc *serverConn) startPush(msg *startPushRequest) { sc.serveG.check() // http://tools.ietf.org/html/rfc7540#section-6.6. @@ -2588,7 +2757,7 @@ func (sc *serverConn) startPush(msg startPushRequest) { // A server that is unable to establish a new stream identifier can send a GOAWAY // frame so that the client is forced to open a new connection for new streams. if sc.maxPushPromiseID+2 >= 1<<31 { - sc.startGracefulShutdown() + sc.startGracefulShutdownInternal() return 0, ErrPushLimitReached } sc.maxPushPromiseID += 2 @@ -2630,7 +2799,7 @@ func (sc *serverConn) startPush(msg startPushRequest) { } // foreachHeaderElement splits v according to the "#rule" construction -// in RFC 2616 section 2.1 and calls fn for each non-empty element. +// in RFC 7230 section 7 and calls fn for each non-empty element. func foreachHeaderElement(v string, fn func(string)) { v = textproto.TrimString(v) if v == "" { @@ -2678,66 +2847,6 @@ func new400Handler(err error) http.HandlerFunc { } } -// ValidTrailerHeader reports whether name is a valid header field name to appear -// in trailers. -// See: http://tools.ietf.org/html/rfc7230#section-4.1.2 -func ValidTrailerHeader(name string) bool { - name = http.CanonicalHeaderKey(name) - if strings.HasPrefix(name, "If-") || badTrailer[name] { - return false - } - return true -} - -var badTrailer = map[string]bool{ - "Authorization": true, - "Cache-Control": true, - "Connection": true, - "Content-Encoding": true, - "Content-Length": true, - "Content-Range": true, - "Content-Type": true, - "Expect": true, - "Host": true, - "Keep-Alive": true, - "Max-Forwards": true, - "Pragma": true, - "Proxy-Authenticate": true, - "Proxy-Authorization": true, - "Proxy-Connection": true, - "Range": true, - "Realm": true, - "Te": true, - "Trailer": true, - "Transfer-Encoding": true, - "Www-Authenticate": true, -} - -// h1ServerShutdownChan returns a channel that will be closed when the -// provided *http.Server wants to shut down. -// -// This is a somewhat hacky way to get at http1 innards. It works -// when the http2 code is bundled into the net/http package in the -// standard library. The alternatives ended up making the cmd/go tool -// depend on http Servers. This is the lightest option for now. -// This is tested via the TestServeShutdown* tests in net/http. -func h1ServerShutdownChan(hs *http.Server) <-chan struct{} { - if fn := testh1ServerShutdownChan; fn != nil { - return fn(hs) - } - var x interface{} = hs - type I interface { - getDoneChan() <-chan struct{} - } - if hs, ok := x.(I); ok { - return hs.getDoneChan() - } - return nil -} - -// optional test hook for h1ServerShutdownChan. -var testh1ServerShutdownChan func(hs *http.Server) <-chan struct{} - // h1ServerKeepAlivesDisabled reports whether hs has its keep-alives // disabled. See comments on h1ServerShutdownChan above for why // the code is written this way. diff --git a/vendor/golang.org/x/net/http2/transport.go b/vendor/golang.org/x/net/http2/transport.go index fef83968670..da91d19dd40 100644 --- a/vendor/golang.org/x/net/http2/transport.go +++ b/vendor/golang.org/x/net/http2/transport.go @@ -18,6 +18,7 @@ import ( "io/ioutil" "log" "math" + mathrand "math/rand" "net" "net/http" "sort" @@ -86,7 +87,7 @@ type Transport struct { // MaxHeaderListSize is the http2 SETTINGS_MAX_HEADER_LIST_SIZE to // send in the initial settings frame. It is how many bytes - // of response headers are allow. Unlike the http2 spec, zero here + // of response headers are allowed. Unlike the http2 spec, zero here // means to use a default limit (currently 10MB). If you actually // want to advertise an ulimited value to the peer, Transport // interprets the highest possible value here (0xffffffff or 1<<32-1) @@ -164,15 +165,17 @@ type ClientConn struct { goAwayDebug string // goAway frame's debug data, retained as a string streams map[uint32]*clientStream // client-initiated nextStreamID uint32 + pendingRequests int // requests blocked and waiting to be sent because len(streams) == maxConcurrentStreams pings map[[8]byte]chan struct{} // in flight ping data to notification channel bw *bufio.Writer br *bufio.Reader fr *Framer lastActive time.Time // Settings from peer: (also guarded by mu) - maxFrameSize uint32 - maxConcurrentStreams uint32 - initialWindowSize uint32 + maxFrameSize uint32 + maxConcurrentStreams uint32 + peerMaxHeaderListSize uint64 + initialWindowSize uint32 hbuf bytes.Buffer // HPACK encoder writes into this henc *hpack.Encoder @@ -216,35 +219,45 @@ type clientStream struct { resTrailer *http.Header // client's Response.Trailer } -// awaitRequestCancel runs in its own goroutine and waits for the user -// to cancel a RoundTrip request, its context to expire, or for the -// request to be done (any way it might be removed from the cc.streams -// map: peer reset, successful completion, TCP connection breakage, -// etc) -func (cs *clientStream) awaitRequestCancel(req *http.Request) { +// awaitRequestCancel waits for the user to cancel a request or for the done +// channel to be signaled. A non-nil error is returned only if the request was +// canceled. +func awaitRequestCancel(req *http.Request, done <-chan struct{}) error { ctx := reqContext(req) if req.Cancel == nil && ctx.Done() == nil { - return + return nil } select { case <-req.Cancel: - cs.cancelStream() - cs.bufPipe.CloseWithError(errRequestCanceled) + return errRequestCanceled case <-ctx.Done(): + return ctx.Err() + case <-done: + return nil + } +} + +// awaitRequestCancel waits for the user to cancel a request, its context to +// expire, or for the request to be done (any way it might be removed from the +// cc.streams map: peer reset, successful completion, TCP connection breakage, +// etc). If the request is canceled, then cs will be canceled and closed. +func (cs *clientStream) awaitRequestCancel(req *http.Request) { + if err := awaitRequestCancel(req, cs.done); err != nil { cs.cancelStream() - cs.bufPipe.CloseWithError(ctx.Err()) - case <-cs.done: + cs.bufPipe.CloseWithError(err) } } func (cs *clientStream) cancelStream() { - cs.cc.mu.Lock() + cc := cs.cc + cc.mu.Lock() didReset := cs.didReset cs.didReset = true - cs.cc.mu.Unlock() + cc.mu.Unlock() if !didReset { - cs.cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) + cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) + cc.forgetStreamID(cs.ID) } } @@ -261,6 +274,13 @@ func (cs *clientStream) checkResetOrDone() error { } } +func (cs *clientStream) getStartedWrite() bool { + cc := cs.cc + cc.mu.Lock() + defer cc.mu.Unlock() + return cs.startedWrite +} + func (cs *clientStream) abortRequestBodyWrite(err error) { if err == nil { panic("nil error") @@ -286,7 +306,26 @@ func (sew stickyErrWriter) Write(p []byte) (n int, err error) { return } -var ErrNoCachedConn = errors.New("http2: no cached connection was available") +// noCachedConnError is the concrete type of ErrNoCachedConn, which +// needs to be detected by net/http regardless of whether it's its +// bundled version (in h2_bundle.go with a rewritten type name) or +// from a user's x/net/http2. As such, as it has a unique method name +// (IsHTTP2NoCachedConnError) that net/http sniffs for via func +// isNoCachedConnError. +type noCachedConnError struct{} + +func (noCachedConnError) IsHTTP2NoCachedConnError() {} +func (noCachedConnError) Error() string { return "http2: no cached connection was available" } + +// isNoCachedConnError reports whether err is of type noCachedConnError +// or its equivalent renamed type in net/http2's h2_bundle.go. Both types +// may coexist in the same running program. +func isNoCachedConnError(err error) bool { + _, ok := err.(interface{ IsHTTP2NoCachedConnError() }) + return ok +} + +var ErrNoCachedConn error = noCachedConnError{} // RoundTripOpt are options for the Transport.RoundTripOpt method. type RoundTripOpt struct { @@ -329,17 +368,28 @@ func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Res } addr := authorityAddr(req.URL.Scheme, req.URL.Host) - for { + for retry := 0; ; retry++ { cc, err := t.connPool().GetClientConn(req, addr) if err != nil { t.vlogf("http2: Transport failed to get client conn for %s: %v", addr, err) return nil, err } traceGotConn(req, cc) - res, err := cc.RoundTrip(req) - if err != nil { - if req, err = shouldRetryRequest(req, err); err == nil { - continue + res, gotErrAfterReqBodyWrite, err := cc.roundTrip(req) + if err != nil && retry <= 6 { + if req, err = shouldRetryRequest(req, err, gotErrAfterReqBodyWrite); err == nil { + // After the first retry, do exponential backoff with 10% jitter. + if retry == 0 { + continue + } + backoff := float64(uint(1) << (uint(retry) - 1)) + backoff += backoff * (0.1 * mathrand.Float64()) + select { + case <-time.After(time.Second * time.Duration(backoff)): + continue + case <-reqContext(req).Done(): + return nil, reqContext(req).Err() + } } } if err != nil { @@ -360,43 +410,50 @@ func (t *Transport) CloseIdleConnections() { } var ( - errClientConnClosed = errors.New("http2: client conn is closed") - errClientConnUnusable = errors.New("http2: client conn not usable") - - errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY") - errClientConnGotGoAwayAfterSomeReqBody = errors.New("http2: Transport received Server's graceful shutdown GOAWAY; some request body already written") + errClientConnClosed = errors.New("http2: client conn is closed") + errClientConnUnusable = errors.New("http2: client conn not usable") + errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY") ) // shouldRetryRequest is called by RoundTrip when a request fails to get // response headers. It is always called with a non-nil error. // It returns either a request to retry (either the same request, or a // modified clone), or an error if the request can't be replayed. -func shouldRetryRequest(req *http.Request, err error) (*http.Request, error) { - switch err { - default: +func shouldRetryRequest(req *http.Request, err error, afterBodyWrite bool) (*http.Request, error) { + if !canRetryError(err) { return nil, err - case errClientConnUnusable, errClientConnGotGoAway: + } + if !afterBodyWrite { return req, nil - case errClientConnGotGoAwayAfterSomeReqBody: - // If the Body is nil (or http.NoBody), it's safe to reuse - // this request and its Body. - if req.Body == nil || reqBodyIsNoBody(req.Body) { - return req, nil - } - // Otherwise we depend on the Request having its GetBody - // func defined. - getBody := reqGetBody(req) // Go 1.8: getBody = req.GetBody - if getBody == nil { - return nil, errors.New("http2: Transport: peer server initiated graceful shutdown after some of Request.Body was written; define Request.GetBody to avoid this error") - } - body, err := getBody() - if err != nil { - return nil, err - } - newReq := *req - newReq.Body = body - return &newReq, nil } + // If the Body is nil (or http.NoBody), it's safe to reuse + // this request and its Body. + if req.Body == nil || reqBodyIsNoBody(req.Body) { + return req, nil + } + // Otherwise we depend on the Request having its GetBody + // func defined. + getBody := reqGetBody(req) // Go 1.8: getBody = req.GetBody + if getBody == nil { + return nil, fmt.Errorf("http2: Transport: cannot retry err [%v] after Request.Body was written; define Request.GetBody to avoid this error", err) + } + body, err := getBody() + if err != nil { + return nil, err + } + newReq := *req + newReq.Body = body + return &newReq, nil +} + +func canRetryError(err error) bool { + if err == errClientConnUnusable || err == errClientConnGotGoAway { + return true + } + if se, ok := err.(StreamError); ok { + return se.Code == ErrCodeRefusedStream + } + return false } func (t *Transport) dialClientConn(addr string, singleUse bool) (*ClientConn, error) { @@ -474,17 +531,18 @@ func (t *Transport) NewClientConn(c net.Conn) (*ClientConn, error) { func (t *Transport) newClientConn(c net.Conn, singleUse bool) (*ClientConn, error) { cc := &ClientConn{ - t: t, - tconn: c, - readerDone: make(chan struct{}), - nextStreamID: 1, - maxFrameSize: 16 << 10, // spec default - initialWindowSize: 65535, // spec default - maxConcurrentStreams: 1000, // "infinite", per spec. 1000 seems good enough. - streams: make(map[uint32]*clientStream), - singleUse: singleUse, - wantSettingsAck: true, - pings: make(map[[8]byte]chan struct{}), + t: t, + tconn: c, + readerDone: make(chan struct{}), + nextStreamID: 1, + maxFrameSize: 16 << 10, // spec default + initialWindowSize: 65535, // spec default + maxConcurrentStreams: 1000, // "infinite", per spec. 1000 seems good enough. + peerMaxHeaderListSize: 0xffffffffffffffff, // "infinite", per spec. Use 2^64-1 instead. + streams: make(map[uint32]*clientStream), + singleUse: singleUse, + wantSettingsAck: true, + pings: make(map[[8]byte]chan struct{}), } if d := t.idleConnTimeout(); d != 0 { cc.idleTimeout = d @@ -560,6 +618,8 @@ func (cc *ClientConn) setGoAway(f *GoAwayFrame) { } } +// CanTakeNewRequest reports whether the connection can take a new request, +// meaning it has not been closed or received or sent a GOAWAY. func (cc *ClientConn) CanTakeNewRequest() bool { cc.mu.Lock() defer cc.mu.Unlock() @@ -571,8 +631,7 @@ func (cc *ClientConn) canTakeNewRequestLocked() bool { return false } return cc.goAway == nil && !cc.closed && - int64(len(cc.streams)+1) < int64(cc.maxConcurrentStreams) && - cc.nextStreamID < math.MaxInt32 + int64(cc.nextStreamID)+int64(cc.pendingRequests) < math.MaxInt32 } // onIdleTimeout is called from a time.AfterFunc goroutine. It will @@ -694,7 +753,7 @@ func checkConnHeaders(req *http.Request) error { // req.ContentLength, where 0 actually means zero (not unknown) and -1 // means unknown. func actualContentLength(req *http.Request) int64 { - if req.Body == nil { + if req.Body == nil || reqBodyIsNoBody(req.Body) { return 0 } if req.ContentLength != 0 { @@ -704,8 +763,13 @@ func actualContentLength(req *http.Request) int64 { } func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { + resp, _, err := cc.roundTrip(req) + return resp, err +} + +func (cc *ClientConn) roundTrip(req *http.Request) (res *http.Response, gotErrAfterReqBodyWrite bool, err error) { if err := checkConnHeaders(req); err != nil { - return nil, err + return nil, false, err } if cc.idleTimer != nil { cc.idleTimer.Stop() @@ -713,20 +777,19 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { trailers, err := commaSeparatedTrailers(req) if err != nil { - return nil, err + return nil, false, err } hasTrailers := trailers != "" cc.mu.Lock() - cc.lastActive = time.Now() - if cc.closed || !cc.canTakeNewRequestLocked() { + if err := cc.awaitOpenSlotForRequest(req); err != nil { cc.mu.Unlock() - return nil, errClientConnUnusable + return nil, false, err } body := req.Body - hasBody := body != nil contentLen := actualContentLength(req) + hasBody := contentLen != 0 // TODO(bradfitz): this is a copy of the logic in net/http. Unify somewhere? var requestedGzip bool @@ -755,7 +818,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { hdrs, err := cc.encodeHeaders(req, requestedGzip, trailers, contentLen) if err != nil { cc.mu.Unlock() - return nil, err + return nil, false, err } cs := cc.newStream() @@ -767,7 +830,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { cc.wmu.Lock() endStream := !hasBody && !hasTrailers - werr := cc.writeHeaders(cs.ID, endStream, hdrs) + werr := cc.writeHeaders(cs.ID, endStream, int(cc.maxFrameSize), hdrs) cc.wmu.Unlock() traceWroteHeaders(cs.trace) cc.mu.Unlock() @@ -781,7 +844,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { // Don't bother sending a RST_STREAM (our write already failed; // no need to keep writing) traceWroteRequest(cs.trace, werr) - return nil, werr + return nil, false, werr } var respHeaderTimer <-chan time.Time @@ -800,7 +863,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { bodyWritten := false ctx := reqContext(req) - handleReadLoopResponse := func(re resAndError) (*http.Response, error) { + handleReadLoopResponse := func(re resAndError) (*http.Response, bool, error) { res := re.res if re.err != nil || res.StatusCode > 299 { // On error or status code 3xx, 4xx, 5xx, etc abort any @@ -816,19 +879,12 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { cs.abortRequestBodyWrite(errStopReqBodyWrite) } if re.err != nil { - if re.err == errClientConnGotGoAway { - cc.mu.Lock() - if cs.startedWrite { - re.err = errClientConnGotGoAwayAfterSomeReqBody - } - cc.mu.Unlock() - } cc.forgetStreamID(cs.ID) - return nil, re.err + return nil, cs.getStartedWrite(), re.err } res.Request = req res.TLS = cc.tlsState - return res, nil + return res, false, nil } for { @@ -836,37 +892,37 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { case re := <-readLoopResCh: return handleReadLoopResponse(re) case <-respHeaderTimer: - cc.forgetStreamID(cs.ID) if !hasBody || bodyWritten { cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) } else { bodyWriter.cancel() cs.abortRequestBodyWrite(errStopReqBodyWriteAndCancel) } - return nil, errTimeout - case <-ctx.Done(): cc.forgetStreamID(cs.ID) + return nil, cs.getStartedWrite(), errTimeout + case <-ctx.Done(): if !hasBody || bodyWritten { cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) } else { bodyWriter.cancel() cs.abortRequestBodyWrite(errStopReqBodyWriteAndCancel) } - return nil, ctx.Err() - case <-req.Cancel: cc.forgetStreamID(cs.ID) + return nil, cs.getStartedWrite(), ctx.Err() + case <-req.Cancel: if !hasBody || bodyWritten { cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) } else { bodyWriter.cancel() cs.abortRequestBodyWrite(errStopReqBodyWriteAndCancel) } - return nil, errRequestCanceled + cc.forgetStreamID(cs.ID) + return nil, cs.getStartedWrite(), errRequestCanceled case <-cs.peerReset: // processResetStream already removed the // stream from the streams map; no need for // forgetStreamID. - return nil, cs.resetErr + return nil, cs.getStartedWrite(), cs.resetErr case err := <-bodyWriter.resc: // Prefer the read loop's response, if available. Issue 16102. select { @@ -875,7 +931,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { default: } if err != nil { - return nil, err + return nil, cs.getStartedWrite(), err } bodyWritten = true if d := cc.responseHeaderTimeout(); d != 0 { @@ -887,14 +943,55 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { } } +// awaitOpenSlotForRequest waits until len(streams) < maxConcurrentStreams. +// Must hold cc.mu. +func (cc *ClientConn) awaitOpenSlotForRequest(req *http.Request) error { + var waitingForConn chan struct{} + var waitingForConnErr error // guarded by cc.mu + for { + cc.lastActive = time.Now() + if cc.closed || !cc.canTakeNewRequestLocked() { + if waitingForConn != nil { + close(waitingForConn) + } + return errClientConnUnusable + } + if int64(len(cc.streams))+1 <= int64(cc.maxConcurrentStreams) { + if waitingForConn != nil { + close(waitingForConn) + } + return nil + } + // Unfortunately, we cannot wait on a condition variable and channel at + // the same time, so instead, we spin up a goroutine to check if the + // request is canceled while we wait for a slot to open in the connection. + if waitingForConn == nil { + waitingForConn = make(chan struct{}) + go func() { + if err := awaitRequestCancel(req, waitingForConn); err != nil { + cc.mu.Lock() + waitingForConnErr = err + cc.cond.Broadcast() + cc.mu.Unlock() + } + }() + } + cc.pendingRequests++ + cc.cond.Wait() + cc.pendingRequests-- + if waitingForConnErr != nil { + return waitingForConnErr + } + } +} + // requires cc.wmu be held -func (cc *ClientConn) writeHeaders(streamID uint32, endStream bool, hdrs []byte) error { +func (cc *ClientConn) writeHeaders(streamID uint32, endStream bool, maxFrameSize int, hdrs []byte) error { first := true // first frame written (HEADERS is first, then CONTINUATION) - frameSize := int(cc.maxFrameSize) for len(hdrs) > 0 && cc.werr == nil { chunk := hdrs - if len(chunk) > frameSize { - chunk = chunk[:frameSize] + if len(chunk) > maxFrameSize { + chunk = chunk[:maxFrameSize] } hdrs = hdrs[len(chunk):] endHeaders := len(hdrs) == 0 @@ -1002,17 +1099,26 @@ func (cs *clientStream) writeRequestBody(body io.Reader, bodyCloser io.Closer) ( var trls []byte if hasTrailers { cc.mu.Lock() - defer cc.mu.Unlock() - trls = cc.encodeTrailers(req) + trls, err = cc.encodeTrailers(req) + cc.mu.Unlock() + if err != nil { + cc.writeStreamReset(cs.ID, ErrCodeInternal, err) + cc.forgetStreamID(cs.ID) + return err + } } + cc.mu.Lock() + maxFrameSize := int(cc.maxFrameSize) + cc.mu.Unlock() + cc.wmu.Lock() defer cc.wmu.Unlock() // Two ways to send END_STREAM: either with trailers, or // with an empty DATA frame. if len(trls) > 0 { - err = cc.writeHeaders(cs.ID, true, trls) + err = cc.writeHeaders(cs.ID, true, maxFrameSize, trls) } else { err = cc.fr.WriteData(cs.ID, true, nil) } @@ -1106,62 +1212,86 @@ func (cc *ClientConn) encodeHeaders(req *http.Request, addGzipHeader bool, trail } } - // 8.1.2.3 Request Pseudo-Header Fields - // The :path pseudo-header field includes the path and query parts of the - // target URI (the path-absolute production and optionally a '?' character - // followed by the query production (see Sections 3.3 and 3.4 of - // [RFC3986]). - cc.writeHeader(":authority", host) - cc.writeHeader(":method", req.Method) - if req.Method != "CONNECT" { - cc.writeHeader(":path", path) - cc.writeHeader(":scheme", req.URL.Scheme) - } - if trailers != "" { - cc.writeHeader("trailer", trailers) - } + enumerateHeaders := func(f func(name, value string)) { + // 8.1.2.3 Request Pseudo-Header Fields + // The :path pseudo-header field includes the path and query parts of the + // target URI (the path-absolute production and optionally a '?' character + // followed by the query production (see Sections 3.3 and 3.4 of + // [RFC3986]). + f(":authority", host) + f(":method", req.Method) + if req.Method != "CONNECT" { + f(":path", path) + f(":scheme", req.URL.Scheme) + } + if trailers != "" { + f("trailer", trailers) + } - var didUA bool - for k, vv := range req.Header { - lowKey := strings.ToLower(k) - switch lowKey { - case "host", "content-length": - // Host is :authority, already sent. - // Content-Length is automatic, set below. - continue - case "connection", "proxy-connection", "transfer-encoding", "upgrade", "keep-alive": - // Per 8.1.2.2 Connection-Specific Header - // Fields, don't send connection-specific - // fields. We have already checked if any - // are error-worthy so just ignore the rest. - continue - case "user-agent": - // Match Go's http1 behavior: at most one - // User-Agent. If set to nil or empty string, - // then omit it. Otherwise if not mentioned, - // include the default (below). - didUA = true - if len(vv) < 1 { + var didUA bool + for k, vv := range req.Header { + if strings.EqualFold(k, "host") || strings.EqualFold(k, "content-length") { + // Host is :authority, already sent. + // Content-Length is automatic, set below. continue - } - vv = vv[:1] - if vv[0] == "" { + } else if strings.EqualFold(k, "connection") || strings.EqualFold(k, "proxy-connection") || + strings.EqualFold(k, "transfer-encoding") || strings.EqualFold(k, "upgrade") || + strings.EqualFold(k, "keep-alive") { + // Per 8.1.2.2 Connection-Specific Header + // Fields, don't send connection-specific + // fields. We have already checked if any + // are error-worthy so just ignore the rest. continue + } else if strings.EqualFold(k, "user-agent") { + // Match Go's http1 behavior: at most one + // User-Agent. If set to nil or empty string, + // then omit it. Otherwise if not mentioned, + // include the default (below). + didUA = true + if len(vv) < 1 { + continue + } + vv = vv[:1] + if vv[0] == "" { + continue + } + + } + + for _, v := range vv { + f(k, v) } } - for _, v := range vv { - cc.writeHeader(lowKey, v) + if shouldSendReqContentLength(req.Method, contentLength) { + f("content-length", strconv.FormatInt(contentLength, 10)) + } + if addGzipHeader { + f("accept-encoding", "gzip") + } + if !didUA { + f("user-agent", defaultUserAgent) } } - if shouldSendReqContentLength(req.Method, contentLength) { - cc.writeHeader("content-length", strconv.FormatInt(contentLength, 10)) - } - if addGzipHeader { - cc.writeHeader("accept-encoding", "gzip") - } - if !didUA { - cc.writeHeader("user-agent", defaultUserAgent) + + // Do a first pass over the headers counting bytes to ensure + // we don't exceed cc.peerMaxHeaderListSize. This is done as a + // separate pass before encoding the headers to prevent + // modifying the hpack state. + hlSize := uint64(0) + enumerateHeaders(func(name, value string) { + hf := hpack.HeaderField{Name: name, Value: value} + hlSize += uint64(hf.Size()) + }) + + if hlSize > cc.peerMaxHeaderListSize { + return nil, errRequestHeaderListSize } + + // Header list size is ok. Write the headers. + enumerateHeaders(func(name, value string) { + cc.writeHeader(strings.ToLower(name), value) + }) + return cc.hbuf.Bytes(), nil } @@ -1188,17 +1318,29 @@ func shouldSendReqContentLength(method string, contentLength int64) bool { } // requires cc.mu be held. -func (cc *ClientConn) encodeTrailers(req *http.Request) []byte { +func (cc *ClientConn) encodeTrailers(req *http.Request) ([]byte, error) { cc.hbuf.Reset() + + hlSize := uint64(0) + for k, vv := range req.Trailer { + for _, v := range vv { + hf := hpack.HeaderField{Name: k, Value: v} + hlSize += uint64(hf.Size()) + } + } + if hlSize > cc.peerMaxHeaderListSize { + return nil, errRequestHeaderListSize + } + for k, vv := range req.Trailer { - // Transfer-Encoding, etc.. have already been filter at the + // Transfer-Encoding, etc.. have already been filtered at the // start of RoundTrip lowKey := strings.ToLower(k) for _, v := range vv { cc.writeHeader(lowKey, v) } } - return cc.hbuf.Bytes() + return cc.hbuf.Bytes(), nil } func (cc *ClientConn) writeHeader(name, value string) { @@ -1246,7 +1388,9 @@ func (cc *ClientConn) streamByID(id uint32, andRemove bool) *clientStream { cc.idleTimer.Reset(cc.idleTimeout) } close(cs.done) - cc.cond.Broadcast() // wake up checkResetOrDone via clientStream.awaitFlowControl + // Wake up checkResetOrDone via clientStream.awaitFlowControl and + // wake up RoundTrip if there is a pending request. + cc.cond.Broadcast() } return cs } @@ -1254,17 +1398,12 @@ func (cc *ClientConn) streamByID(id uint32, andRemove bool) *clientStream { // clientConnReadLoop is the state owned by the clientConn's frame-reading readLoop. type clientConnReadLoop struct { cc *ClientConn - activeRes map[uint32]*clientStream // keyed by streamID closeWhenIdle bool } // readLoop runs in its own goroutine and reads and dispatches frames. func (cc *ClientConn) readLoop() { - rl := &clientConnReadLoop{ - cc: cc, - activeRes: make(map[uint32]*clientStream), - } - + rl := &clientConnReadLoop{cc: cc} defer rl.cleanup() cc.readerErr = rl.run() if ce, ok := cc.readerErr.(ConnectionError); ok { @@ -1319,10 +1458,8 @@ func (rl *clientConnReadLoop) cleanup() { } else if err == io.EOF { err = io.ErrUnexpectedEOF } - for _, cs := range rl.activeRes { - cs.bufPipe.CloseWithError(err) - } for _, cs := range cc.streams { + cs.bufPipe.CloseWithError(err) // no-op if already closed select { case cs.resc <- resAndError{err: err}: default: @@ -1345,8 +1482,9 @@ func (rl *clientConnReadLoop) run() error { cc.vlogf("http2: Transport readFrame error on conn %p: (%T) %v", cc, err, err) } if se, ok := err.(StreamError); ok { - if cs := cc.streamByID(se.StreamID, true /*ended; remove it*/); cs != nil { + if cs := cc.streamByID(se.StreamID, false); cs != nil { cs.cc.writeStreamReset(cs.ID, se.Code, err) + cs.cc.forgetStreamID(cs.ID) if se.Cause == nil { se.Cause = cc.fr.errDetail } @@ -1399,7 +1537,7 @@ func (rl *clientConnReadLoop) run() error { } return err } - if rl.closeWhenIdle && gotReply && maybeIdle && len(rl.activeRes) == 0 { + if rl.closeWhenIdle && gotReply && maybeIdle { cc.closeIfIdle() } } @@ -1407,13 +1545,31 @@ func (rl *clientConnReadLoop) run() error { func (rl *clientConnReadLoop) processHeaders(f *MetaHeadersFrame) error { cc := rl.cc - cs := cc.streamByID(f.StreamID, f.StreamEnded()) + cs := cc.streamByID(f.StreamID, false) if cs == nil { // We'd get here if we canceled a request while the // server had its response still in flight. So if this // was just something we canceled, ignore it. return nil } + if f.StreamEnded() { + // Issue 20521: If the stream has ended, streamByID() causes + // clientStream.done to be closed, which causes the request's bodyWriter + // to be closed with an errStreamClosed, which may be received by + // clientConn.RoundTrip before the result of processing these headers. + // Deferring stream closure allows the header processing to occur first. + // clientConn.RoundTrip may still receive the bodyWriter error first, but + // the fix for issue 16102 prioritises any response. + // + // Issue 22413: If there is no request body, we should close the + // stream before writing to cs.resc so that the stream is closed + // immediately once RoundTrip returns. + if cs.req.Body != nil { + defer cc.forgetStreamID(f.StreamID) + } else { + cc.forgetStreamID(f.StreamID) + } + } if !cs.firstByte { if cs.trace != nil { // TODO(bradfitz): move first response byte earlier, @@ -1437,6 +1593,7 @@ func (rl *clientConnReadLoop) processHeaders(f *MetaHeadersFrame) error { } // Any other error type is a stream error. cs.cc.writeStreamReset(f.StreamID, ErrCodeProtocol, err) + cc.forgetStreamID(cs.ID) cs.resc <- resAndError{err: err} return nil // return nil from process* funcs to keep conn alive } @@ -1444,9 +1601,6 @@ func (rl *clientConnReadLoop) processHeaders(f *MetaHeadersFrame) error { // (nil, nil) special case. See handleResponse docs. return nil } - if res.Body != noBody { - rl.activeRes[cs.ID] = cs - } cs.resTrailer = &res.Trailer cs.resc <- resAndError{res: res} return nil @@ -1466,11 +1620,11 @@ func (rl *clientConnReadLoop) handleResponse(cs *clientStream, f *MetaHeadersFra status := f.PseudoValue("status") if status == "" { - return nil, errors.New("missing status pseudo header") + return nil, errors.New("malformed response from server: missing status pseudo header") } statusCode, err := strconv.Atoi(status) if err != nil { - return nil, errors.New("malformed non-numeric status pseudo header") + return nil, errors.New("malformed response from server: malformed non-numeric status pseudo header") } if statusCode == 100 { @@ -1528,8 +1682,7 @@ func (rl *clientConnReadLoop) handleResponse(cs *clientStream, f *MetaHeadersFra return res, nil } - buf := new(bytes.Buffer) // TODO(bradfitz): recycle this garbage - cs.bufPipe = pipe{b: buf} + cs.bufPipe = pipe{b: &dataBuffer{expected: res.ContentLength}} cs.bytesRemain = res.ContentLength res.Body = transportResponseBody{cs} go cs.awaitRequestCancel(cs.req) @@ -1656,6 +1809,7 @@ func (b transportResponseBody) Close() error { cc.wmu.Lock() if !serverSentStreamEnd { cc.fr.WriteRSTStream(cs.ID, ErrCodeCancel) + cs.didReset = true } // Return connection-level flow control. if unread > 0 { @@ -1668,6 +1822,7 @@ func (b transportResponseBody) Close() error { } cs.bufPipe.BreakWithError(errClosedResponseBody) + cc.forgetStreamID(cs.ID) return nil } @@ -1702,13 +1857,23 @@ func (rl *clientConnReadLoop) processData(f *DataFrame) error { } return nil } + if !cs.firstByte { + cc.logf("protocol error: received DATA before a HEADERS frame") + rl.endStreamError(cs, StreamError{ + StreamID: f.StreamID, + Code: ErrCodeProtocol, + }) + return nil + } if f.Length > 0 { - if len(data) > 0 && cs.bufPipe.b == nil { - // Data frame after it's already closed? - cc.logf("http2: Transport received DATA frame for closed stream; closing connection") - return ConnectionError(ErrCodeProtocol) + if cs.req.Method == "HEAD" && len(data) > 0 { + cc.logf("protocol error: received DATA on a HEAD request") + rl.endStreamError(cs, StreamError{ + StreamID: f.StreamID, + Code: ErrCodeProtocol, + }) + return nil } - // Check connection-level flow control. cc.mu.Lock() if cs.inflow.available() >= int32(f.Length) { @@ -1719,16 +1884,27 @@ func (rl *clientConnReadLoop) processData(f *DataFrame) error { } // Return any padded flow control now, since we won't // refund it later on body reads. - if pad := int32(f.Length) - int32(len(data)); pad > 0 { - cs.inflow.add(pad) - cc.inflow.add(pad) + var refund int + if pad := int(f.Length) - len(data); pad > 0 { + refund += pad + } + // Return len(data) now if the stream is already closed, + // since data will never be read. + didReset := cs.didReset + if didReset { + refund += len(data) + } + if refund > 0 { + cc.inflow.add(int32(refund)) cc.wmu.Lock() - cc.fr.WriteWindowUpdate(0, uint32(pad)) - cc.fr.WriteWindowUpdate(cs.ID, uint32(pad)) + cc.fr.WriteWindowUpdate(0, uint32(refund)) + if !didReset { + cs.inflow.add(int32(refund)) + cc.fr.WriteWindowUpdate(cs.ID, uint32(refund)) + } cc.bw.Flush() cc.wmu.Unlock() } - didReset := cs.didReset cc.mu.Unlock() if len(data) > 0 && !didReset { @@ -1759,11 +1935,10 @@ func (rl *clientConnReadLoop) endStreamError(cs *clientStream, err error) { err = io.EOF code = cs.copyTrailers } - cs.bufPipe.closeWithErrorAndCode(err, code) - delete(rl.activeRes, cs.ID) if isConnectionCloseRequest(cs.req) { rl.closeWhenIdle = true } + cs.bufPipe.closeWithErrorAndCode(err, code) select { case cs.resc <- resAndError{err: err}: @@ -1811,6 +1986,8 @@ func (rl *clientConnReadLoop) processSettings(f *SettingsFrame) error { cc.maxFrameSize = s.Val case SettingMaxConcurrentStreams: cc.maxConcurrentStreams = s.Val + case SettingMaxHeaderListSize: + cc.peerMaxHeaderListSize = uint64(s.Val) case SettingInitialWindowSize: // Values above the maximum flow-control // window size of 2^31-1 MUST be treated as a @@ -1888,7 +2065,6 @@ func (rl *clientConnReadLoop) processResetStream(f *RSTStreamFrame) error { cs.bufPipe.CloseWithError(err) cs.cc.cond.Broadcast() // wake up checkResetOrDone via clientStream.awaitFlowControl } - delete(rl.activeRes, cs.ID) return nil } @@ -1977,6 +2153,7 @@ func (cc *ClientConn) writeStreamReset(streamID uint32, code ErrCode, err error) var ( errResponseHeaderListSize = errors.New("http2: response header list larger than advertised limit") + errRequestHeaderListSize = errors.New("http2: request header list larger than peer's advertised limit") errPseudoTrailers = errors.New("http2: invalid pseudo header in trailers") ) diff --git a/vendor/golang.org/x/net/http2/write.go b/vendor/golang.org/x/net/http2/write.go index 6b0dfae319a..54ab4a88e7b 100644 --- a/vendor/golang.org/x/net/http2/write.go +++ b/vendor/golang.org/x/net/http2/write.go @@ -10,7 +10,6 @@ import ( "log" "net/http" "net/url" - "time" "golang.org/x/net/http2/hpack" "golang.org/x/net/lex/httplex" @@ -90,11 +89,7 @@ type writeGoAway struct { func (p *writeGoAway) writeFrame(ctx writeContext) error { err := ctx.Framer().WriteGoAway(p.maxStreamID, p.code, nil) - if p.code != 0 { - ctx.Flush() // ignore error: we're hanging up on them anyway - time.Sleep(50 * time.Millisecond) - ctx.CloseConn() - } + ctx.Flush() // ignore error: we're hanging up on them anyway return err } diff --git a/vendor/golang.org/x/net/http2/writesched_priority.go b/vendor/golang.org/x/net/http2/writesched_priority.go index 01132721bdd..848fed6ec76 100644 --- a/vendor/golang.org/x/net/http2/writesched_priority.go +++ b/vendor/golang.org/x/net/http2/writesched_priority.go @@ -53,7 +53,7 @@ type PriorityWriteSchedulerConfig struct { } // NewPriorityWriteScheduler constructs a WriteScheduler that schedules -// frames by following HTTP/2 priorities as described in RFC 7340 Section 5.3. +// frames by following HTTP/2 priorities as described in RFC 7540 Section 5.3. // If cfg is nil, default options are used. func NewPriorityWriteScheduler(cfg *PriorityWriteSchedulerConfig) WriteScheduler { if cfg == nil { diff --git a/vendor/golang.org/x/net/trace/events.go b/vendor/golang.org/x/net/trace/events.go index d8daec1a791..c646a6952e5 100644 --- a/vendor/golang.org/x/net/trace/events.go +++ b/vendor/golang.org/x/net/trace/events.go @@ -39,9 +39,9 @@ var buckets = []bucket{ } // RenderEvents renders the HTML page typically served at /debug/events. -// It does not do any auth checking; see AuthRequest for the default auth check -// used by the handler registered on http.DefaultServeMux. -// req may be nil. +// It does not do any auth checking. The request may be nil. +// +// Most users will use the Events handler. func RenderEvents(w http.ResponseWriter, req *http.Request, sensitive bool) { now := time.Now() data := &struct { diff --git a/vendor/golang.org/x/net/trace/trace.go b/vendor/golang.org/x/net/trace/trace.go index 64f56a37368..a46ee0eaa31 100644 --- a/vendor/golang.org/x/net/trace/trace.go +++ b/vendor/golang.org/x/net/trace/trace.go @@ -77,7 +77,6 @@ import ( "sync/atomic" "time" - "golang.org/x/net/context" "golang.org/x/net/internal/timeseries" ) @@ -111,30 +110,46 @@ var AuthRequest = func(req *http.Request) (any, sensitive bool) { } func init() { - http.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) { - any, sensitive := AuthRequest(req) - if !any { - http.Error(w, "not allowed", http.StatusUnauthorized) - return - } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - Render(w, req, sensitive) - }) - http.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) { - any, sensitive := AuthRequest(req) - if !any { - http.Error(w, "not allowed", http.StatusUnauthorized) - return - } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - RenderEvents(w, req, sensitive) - }) + // TODO(jbd): Serve Traces from /debug/traces in the future? + // There is no requirement for a request to be present to have traces. + http.HandleFunc("/debug/requests", Traces) + http.HandleFunc("/debug/events", Events) +} + +// Traces responds with traces from the program. +// The package initialization registers it in http.DefaultServeMux +// at /debug/requests. +// +// It performs authorization by running AuthRequest. +func Traces(w http.ResponseWriter, req *http.Request) { + any, sensitive := AuthRequest(req) + if !any { + http.Error(w, "not allowed", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + Render(w, req, sensitive) +} + +// Events responds with a page of events collected by EventLogs. +// The package initialization registers it in http.DefaultServeMux +// at /debug/events. +// +// It performs authorization by running AuthRequest. +func Events(w http.ResponseWriter, req *http.Request) { + any, sensitive := AuthRequest(req) + if !any { + http.Error(w, "not allowed", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + RenderEvents(w, req, sensitive) } // Render renders the HTML page typically served at /debug/requests. -// It does not do any auth checking; see AuthRequest for the default auth check -// used by the handler registered on http.DefaultServeMux. -// req may be nil. +// It does not do any auth checking. The request may be nil. +// +// Most users will use the Traces handler. func Render(w io.Writer, req *http.Request, sensitive bool) { data := &struct { Families []string @@ -271,18 +286,6 @@ type contextKeyT string var contextKey = contextKeyT("golang.org/x/net/trace.Trace") -// NewContext returns a copy of the parent context -// and associates it with a Trace. -func NewContext(ctx context.Context, tr Trace) context.Context { - return context.WithValue(ctx, contextKey, tr) -} - -// FromContext returns the Trace bound to the context, if any. -func FromContext(ctx context.Context) (tr Trace, ok bool) { - tr, ok = ctx.Value(contextKey).(Trace) - return -} - // Trace represents an active request. type Trace interface { // LazyLog adds x to the event log. It will be evaluated each time the @@ -365,7 +368,11 @@ func New(family, title string) Trace { } func (tr *trace) Finish() { - tr.Elapsed = time.Now().Sub(tr.Start) + elapsed := time.Now().Sub(tr.Start) + tr.mu.Lock() + tr.Elapsed = elapsed + tr.mu.Unlock() + if DebugUseAfterFinish { buf := make([]byte, 4<<10) // 4 KB should be enough n := runtime.Stack(buf, false) @@ -378,14 +385,17 @@ func (tr *trace) Finish() { m.Remove(tr) f := getFamily(tr.Family, true) + tr.mu.RLock() // protects tr fields in Cond.match calls for _, b := range f.Buckets { if b.Cond.match(tr) { b.Add(tr) } } + tr.mu.RUnlock() + // Add a sample of elapsed time as microseconds to the family's timeseries h := new(histogram) - h.addMeasurement(tr.Elapsed.Nanoseconds() / 1e3) + h.addMeasurement(elapsed.Nanoseconds() / 1e3) f.LatencyMu.Lock() f.Latency.Add(h) f.LatencyMu.Unlock() @@ -681,25 +691,20 @@ type trace struct { // Title is the title of this trace. Title string - // Timing information. - Start time.Time - Elapsed time.Duration // zero while active - - // Trace information if non-zero. - traceID uint64 - spanID uint64 - - // Whether this trace resulted in an error. - IsError bool + // Start time of the this trace. + Start time.Time - // Append-only sequence of events (modulo discards). mu sync.RWMutex - events []event + events []event // Append-only sequence of events (modulo discards). maxEvents int + recycler func(interface{}) + IsError bool // Whether this trace resulted in an error. + Elapsed time.Duration // Elapsed time for this trace, zero while active. + traceID uint64 // Trace information if non-zero. + spanID uint64 - refs int32 // how many buckets this is in - recycler func(interface{}) - disc discarded // scratch space to avoid allocation + refs int32 // how many buckets this is in + disc discarded // scratch space to avoid allocation finishStack []byte // where finish was called, if DebugUseAfterFinish is set @@ -711,14 +716,18 @@ func (tr *trace) reset() { tr.Family = "" tr.Title = "" tr.Start = time.Time{} + + tr.mu.Lock() tr.Elapsed = 0 tr.traceID = 0 tr.spanID = 0 tr.IsError = false tr.maxEvents = 0 tr.events = nil - tr.refs = 0 tr.recycler = nil + tr.mu.Unlock() + + tr.refs = 0 tr.disc = 0 tr.finishStack = nil for i := range tr.eventsBuf { @@ -798,21 +807,31 @@ func (tr *trace) LazyPrintf(format string, a ...interface{}) { tr.addEvent(&lazySprintf{format, a}, false, false) } -func (tr *trace) SetError() { tr.IsError = true } +func (tr *trace) SetError() { + tr.mu.Lock() + tr.IsError = true + tr.mu.Unlock() +} func (tr *trace) SetRecycler(f func(interface{})) { + tr.mu.Lock() tr.recycler = f + tr.mu.Unlock() } func (tr *trace) SetTraceInfo(traceID, spanID uint64) { + tr.mu.Lock() tr.traceID, tr.spanID = traceID, spanID + tr.mu.Unlock() } func (tr *trace) SetMaxEvents(m int) { + tr.mu.Lock() // Always keep at least three events: first, discarded count, last. if len(tr.events) == 0 && m > 3 { tr.maxEvents = m } + tr.mu.Unlock() } func (tr *trace) ref() { @@ -821,6 +840,7 @@ func (tr *trace) ref() { func (tr *trace) unref() { if atomic.AddInt32(&tr.refs, -1) == 0 { + tr.mu.RLock() if tr.recycler != nil { // freeTrace clears tr, so we hold tr.recycler and tr.events here. go func(f func(interface{}), es []event) { @@ -831,6 +851,7 @@ func (tr *trace) unref() { } }(tr.recycler, tr.events) } + tr.mu.RUnlock() freeTrace(tr) } @@ -841,7 +862,10 @@ func (tr *trace) When() string { } func (tr *trace) ElapsedTime() string { + tr.mu.RLock() t := tr.Elapsed + tr.mu.RUnlock() + if t == 0 { // Active trace. t = time.Since(tr.Start) diff --git a/vendor/golang.org/x/net/trace/trace_go16.go b/vendor/golang.org/x/net/trace/trace_go16.go new file mode 100644 index 00000000000..d6081911853 --- /dev/null +++ b/vendor/golang.org/x/net/trace/trace_go16.go @@ -0,0 +1,21 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.7 + +package trace + +import "golang.org/x/net/context" + +// NewContext returns a copy of the parent context +// and associates it with a Trace. +func NewContext(ctx context.Context, tr Trace) context.Context { + return context.WithValue(ctx, contextKey, tr) +} + +// FromContext returns the Trace bound to the context, if any. +func FromContext(ctx context.Context) (tr Trace, ok bool) { + tr, ok = ctx.Value(contextKey).(Trace) + return +} diff --git a/vendor/golang.org/x/net/trace/trace_go17.go b/vendor/golang.org/x/net/trace/trace_go17.go new file mode 100644 index 00000000000..df6e1fba7ca --- /dev/null +++ b/vendor/golang.org/x/net/trace/trace_go17.go @@ -0,0 +1,21 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.7 + +package trace + +import "context" + +// NewContext returns a copy of the parent context +// and associates it with a Trace. +func NewContext(ctx context.Context, tr Trace) context.Context { + return context.WithValue(ctx, contextKey, tr) +} + +// FromContext returns the Trace bound to the context, if any. +func FromContext(ctx context.Context) (tr Trace, ok bool) { + tr, ok = ctx.Value(contextKey).(Trace) + return +} diff --git a/vendor/google.golang.org/grpc/AUTHORS b/vendor/google.golang.org/grpc/AUTHORS new file mode 100644 index 00000000000..e491a9e7f78 --- /dev/null +++ b/vendor/google.golang.org/grpc/AUTHORS @@ -0,0 +1 @@ +Google Inc. diff --git a/vendor/google.golang.org/grpc/CONTRIBUTING.md b/vendor/google.golang.org/grpc/CONTRIBUTING.md index 36cd6f7581b..0863eb26b60 100644 --- a/vendor/google.golang.org/grpc/CONTRIBUTING.md +++ b/vendor/google.golang.org/grpc/CONTRIBUTING.md @@ -1,46 +1,36 @@ # How to contribute -We definitely welcome patches and contribution to grpc! Here are some guidelines -and information about how to do so. +We definitely welcome your patches and contributions to gRPC! -## Sending patches - -### Getting started - -1. Check out the code: - - $ go get google.golang.org/grpc - $ cd $GOPATH/src/google.golang.org/grpc - -1. Create a fork of the grpc-go repository. -1. Add your fork as a remote: - - $ git remote add fork git@github.com:$YOURGITHUBUSERNAME/grpc-go.git - -1. Make changes, commit them. -1. Run the test suite: - - $ make test - -1. Push your changes to your fork: - - $ git push fork ... - -1. Open a pull request. +If you are new to github, please start by reading [Pull Request howto](https://help.github.com/articles/about-pull-requests/) ## Legal requirements In order to protect both you and ourselves, you will need to sign the -[Contributor License Agreement](https://cla.developers.google.com/clas). - -## Filing Issues -When filing an issue, make sure to answer these five questions: - -1. What version of Go are you using (`go version`)? -2. What operating system and processor architecture are you using? -3. What did you do? -4. What did you expect to see? -5. What did you see instead? - -### Contributing code -Unless otherwise noted, the Go source files are distributed under the BSD-style license found in the LICENSE file. +[Contributor License Agreement](https://identity.linuxfoundation.org/projects/cncf). + +## Guidelines for Pull Requests +How to get your contributions merged smoothly and quickly. + +- Create **small PRs** that are narrowly focused on **addressing a single concern**. We often times receive PRs that are trying to fix several things at a time, but only one fix is considered acceptable, nothing gets merged and both author's & review's time is wasted. Create more PRs to address different concerns and everyone will be happy. + +- For speculative changes, consider opening an issue and discussing it first. If you are suggesting a behavioral or API change, consider starting with a [gRFC proposal](https://github.com/grpc/proposal). + +- Provide a good **PR description** as a record of **what** change is being made and **why** it was made. Link to a github issue if it exists. + +- Don't fix code style and formatting unless you are already changing that line to address an issue. PRs with irrelevant changes won't be merged. If you do want to fix formatting or style, do that in a separate PR. + +- Unless your PR is trivial, you should expect there will be reviewer comments that you'll need to address before merging. We expect you to be reasonably responsive to those comments, otherwise the PR will be closed after 2-3 weeks of inactivity. + +- Maintain **clean commit history** and use **meaningful commit messages**. PRs with messy commit history are difficult to review and won't be merged. Use `rebase -i upstream/master` to curate your commit history and/or to bring in latest changes from master (but avoid rebasing in the middle of a code review). + +- Keep your PR up to date with upstream/master (if there are merge conflicts, we can't really merge your change). + +- **All tests need to be passing** before your change can be merged. We recommend you **run tests locally** before creating your PR to catch breakages early on. + - `make all` to test everything, OR + - `make vet` to catch vet errors + - `make test` to run the tests + - `make testrace` to run tests in race mode + +- Exceptions to the rules can be made if there's a compelling reason for doing so. + diff --git a/vendor/google.golang.org/grpc/LICENSE b/vendor/google.golang.org/grpc/LICENSE index f4988b45079..d6456956733 100644 --- a/vendor/google.golang.org/grpc/LICENSE +++ b/vendor/google.golang.org/grpc/LICENSE @@ -1,28 +1,202 @@ -Copyright 2014, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/google.golang.org/grpc/Makefile b/vendor/google.golang.org/grpc/Makefile index 03bb01f0b35..6f393a808df 100644 --- a/vendor/google.golang.org/grpc/Makefile +++ b/vendor/google.golang.org/grpc/Makefile @@ -1,4 +1,4 @@ -all: test testrace +all: vet test testrace deps: go get -d -v google.golang.org/grpc/... @@ -20,24 +20,20 @@ proto: echo "error: protoc not installed" >&2; \ exit 1; \ fi - go get -u -v github.com/golang/protobuf/protoc-gen-go - # use $$dir as the root for all proto files in the same directory - for dir in $$(git ls-files '*.proto' | xargs -n1 dirname | uniq); do \ - protoc -I $$dir --go_out=plugins=grpc:$$dir $$dir/*.proto; \ - done + go generate google.golang.org/grpc/... + +vet: + ./vet.sh test: testdeps - go test -v -cpu 1,4 google.golang.org/grpc/... + go test -cpu 1,4 -timeout 5m google.golang.org/grpc/... testrace: testdeps - go test -v -race -cpu 1,4 google.golang.org/grpc/... + go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/... clean: go clean -i google.golang.org/grpc/... -coverage: testdeps - ./coverage.sh --coveralls - .PHONY: \ all \ deps \ @@ -46,7 +42,7 @@ coverage: testdeps updatetestdeps \ build \ proto \ + vet \ test \ testrace \ - clean \ - coverage + clean diff --git a/vendor/google.golang.org/grpc/PATENTS b/vendor/google.golang.org/grpc/PATENTS deleted file mode 100644 index 69b47959fab..00000000000 --- a/vendor/google.golang.org/grpc/PATENTS +++ /dev/null @@ -1,22 +0,0 @@ -Additional IP Rights Grant (Patents) - -"This implementation" means the copyrightable works distributed by -Google as part of the gRPC project. - -Google hereby grants to You a perpetual, worldwide, non-exclusive, -no-charge, royalty-free, irrevocable (except as stated in this section) -patent license to make, have made, use, offer to sell, sell, import, -transfer and otherwise run, modify and propagate the contents of this -implementation of gRPC, where such license applies only to those patent -claims, both currently owned or controlled by Google and acquired in -the future, licensable by Google that are necessarily infringed by this -implementation of gRPC. This grant does not include claims that would be -infringed only as a consequence of further modification of this -implementation. If you or your agent or exclusive licensee institute or -order or agree to the institution of patent litigation against any -entity (including a cross-claim or counterclaim in a lawsuit) alleging -that this implementation of gRPC or any code incorporated within this -implementation of gRPC constitutes direct or contributory patent -infringement, or inducement of patent infringement, then any patent -rights granted to you under this License for this implementation of gRPC -shall terminate as of the date such litigation is filed. diff --git a/vendor/google.golang.org/grpc/README.md b/vendor/google.golang.org/grpc/README.md index 39120c2031c..789adfd6536 100644 --- a/vendor/google.golang.org/grpc/README.md +++ b/vendor/google.golang.org/grpc/README.md @@ -1,8 +1,8 @@ -#gRPC-Go +# gRPC-Go -[![Build Status](https://travis-ci.org/grpc/grpc-go.svg)](https://travis-ci.org/grpc/grpc-go) [![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)](https://godoc.org/google.golang.org/grpc) +[![Build Status](https://travis-ci.org/grpc/grpc-go.svg)](https://travis-ci.org/grpc/grpc-go) [![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)](https://godoc.org/google.golang.org/grpc) [![GoReportCard](https://goreportcard.com/badge/grpc/grpc-go)](https://goreportcard.com/report/github.com/grpc/grpc-go) -The Go implementation of [gRPC](http://www.grpc.io/): A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. For more information see the [gRPC Quick Start](http://www.grpc.io/docs/) guide. +The Go implementation of [gRPC](https://grpc.io/): A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. For more information see the [gRPC Quick Start: Go](https://grpc.io/docs/quickstart/go.html) guide. Installation ------------ @@ -10,29 +10,13 @@ Installation To install this package, you need to install Go and setup your Go workspace on your computer. The simplest way to install the library is to run: ``` -$ go get google.golang.org/grpc +$ go get -u google.golang.org/grpc ``` Prerequisites ------------- -This requires Go 1.5 or later. - -A note on the version used: significant performance improvements in benchmarks -of grpc-go have been seen by upgrading the go version from 1.5 to the latest -1.7.1. - -From https://golang.org/doc/install, one way to install the latest version of go is: -``` -$ GO_VERSION=1.7.1 -$ OS=linux -$ ARCH=amd64 -$ curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz -$ sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz -$ # Put go on the PATH, keep the usual installation dir -$ sudo ln -s /usr/local/go/bin/go /usr/bin/go -$ rm go$GO_VERSION.$OS-$ARCH.tar.gz -``` +This requires Go 1.6 or later. Go 1.7 will be required soon. Constraints ----------- @@ -42,9 +26,13 @@ Documentation ------------- See [API documentation](https://godoc.org/google.golang.org/grpc) for package and API descriptions and find examples in the [examples directory](examples/). +Performance +----------- +See the current benchmarks for some of the languages supported in [this dashboard](https://performance-dot-grpc-testing.appspot.com/explore?dashboard=5652536396611584&widget=490377658&container=1286539696). + Status ------ -GA +General Availability [Google Cloud Platform Launch Stages](https://cloud.google.com/terms/launch-stages). FAQ --- diff --git a/vendor/google.golang.org/grpc/backoff.go b/vendor/google.golang.org/grpc/backoff.go index c99024ee302..c40facce510 100644 --- a/vendor/google.golang.org/grpc/backoff.go +++ b/vendor/google.golang.org/grpc/backoff.go @@ -1,3 +1,21 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package grpc import ( @@ -7,14 +25,12 @@ import ( // DefaultBackoffConfig uses values specified for backoff in // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. -var ( - DefaultBackoffConfig = BackoffConfig{ - MaxDelay: 120 * time.Second, - baseDelay: 1.0 * time.Second, - factor: 1.6, - jitter: 0.2, - } -) +var DefaultBackoffConfig = BackoffConfig{ + MaxDelay: 120 * time.Second, + baseDelay: 1.0 * time.Second, + factor: 1.6, + jitter: 0.2, +} // backoffStrategy defines the methodology for backing off after a grpc // connection failure. diff --git a/vendor/google.golang.org/grpc/balancer.go b/vendor/google.golang.org/grpc/balancer.go index 9d943fbadae..e1730166cde 100644 --- a/vendor/google.golang.org/grpc/balancer.go +++ b/vendor/google.golang.org/grpc/balancer.go @@ -1,33 +1,18 @@ /* * - * Copyright 2016, Google Inc. - * All rights reserved. + * Copyright 2016 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -35,6 +20,7 @@ package grpc import ( "fmt" + "net" "sync" "golang.org/x/net/context" @@ -42,10 +28,12 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/naming" + "google.golang.org/grpc/status" ) // Address represents a server the client connects to. -// This is the EXPERIMENTAL API and may be changed or extended in the future. +// +// Deprecated: please use package balancer. type Address struct { // Addr is the server address on which a connection will be established. Addr string @@ -55,15 +43,22 @@ type Address struct { } // BalancerConfig specifies the configurations for Balancer. +// +// Deprecated: please use package balancer. type BalancerConfig struct { // DialCreds is the transport credential the Balancer implementation can // use to dial to a remote load balancer server. The Balancer implementations // can ignore this if it does not need to talk to another party securely. DialCreds credentials.TransportCredentials + // Dialer is the custom dialer the Balancer implementation can use to dial + // to a remote load balancer server. The Balancer implementations + // can ignore this if it doesn't need to talk to remote balancer. + Dialer func(context.Context, string) (net.Conn, error) } // BalancerGetOptions configures a Get call. -// This is the EXPERIMENTAL API and may be changed or extended in the future. +// +// Deprecated: please use package balancer. type BalancerGetOptions struct { // BlockingWait specifies whether Get should block when there is no // connected address. @@ -71,7 +66,8 @@ type BalancerGetOptions struct { } // Balancer chooses network addresses for RPCs. -// This is the EXPERIMENTAL API and may be changed or extended in the future. +// +// Deprecated: please use package balancer. type Balancer interface { // Start does the initialization work to bootstrap a Balancer. For example, // this function may start the name resolution and watch the updates. It will @@ -144,6 +140,8 @@ func downErrorf(timeout, temporary bool, format string, a ...interface{}) downEr // RoundRobin returns a Balancer that selects addresses round-robin. It uses r to watch // the name resolution updates and updates the addresses available correspondingly. +// +// Deprecated: please use package balancer/roundrobin. func RoundRobin(r naming.Resolver) Balancer { return &roundRobin{r: r} } @@ -167,7 +165,7 @@ type roundRobin struct { func (rr *roundRobin) watchAddrUpdates() error { updates, err := rr.w.Next() if err != nil { - grpclog.Printf("grpc: the naming watcher stops working due to %v.\n", err) + grpclog.Warningf("grpc: the naming watcher stops working due to %v.", err) return err } rr.mu.Lock() @@ -183,7 +181,7 @@ func (rr *roundRobin) watchAddrUpdates() error { for _, v := range rr.addrs { if addr == v.addr { exist = true - grpclog.Println("grpc: The name resolver wanted to add an existing address: ", addr) + grpclog.Infoln("grpc: The name resolver wanted to add an existing address: ", addr) break } } @@ -200,7 +198,7 @@ func (rr *roundRobin) watchAddrUpdates() error { } } default: - grpclog.Println("Unknown update.Op ", update.Op) + grpclog.Errorln("Unknown update.Op ", update.Op) } } // Make a copy of rr.addrs and write it onto rr.addrCh so that gRPC internals gets notified. @@ -211,6 +209,10 @@ func (rr *roundRobin) watchAddrUpdates() error { if rr.done { return ErrClientConnClosing } + select { + case <-rr.addrCh: + default: + } rr.addrCh <- open return nil } @@ -233,7 +235,7 @@ func (rr *roundRobin) Start(target string, config BalancerConfig) error { return err } rr.w = w - rr.addrCh = make(chan []Address) + rr.addrCh = make(chan []Address, 1) go func() { for { if err := rr.watchAddrUpdates(); err != nil { @@ -316,7 +318,7 @@ func (rr *roundRobin) Get(ctx context.Context, opts BalancerGetOptions) (addr Ad if !opts.BlockingWait { if len(rr.addrs) == 0 { rr.mu.Unlock() - err = Errorf(codes.Unavailable, "there is no address available") + err = status.Errorf(codes.Unavailable, "there is no address available") return } // Returns the next addr on rr.addrs for failfast RPCs. @@ -385,6 +387,9 @@ func (rr *roundRobin) Notify() <-chan []Address { func (rr *roundRobin) Close() error { rr.mu.Lock() defer rr.mu.Unlock() + if rr.done { + return errBalancerClosed + } rr.done = true if rr.w != nil { rr.w.Close() @@ -398,3 +403,14 @@ func (rr *roundRobin) Close() error { } return nil } + +// pickFirst is used to test multi-addresses in one addrConn in which all addresses share the same addrConn. +// It is a wrapper around roundRobin balancer. The logic of all methods works fine because balancer.Get() +// returns the only address Up by resetTransport(). +type pickFirst struct { + *roundRobin +} + +func pickFirstBalancerV1(r naming.Resolver) Balancer { + return &pickFirst{&roundRobin{r: r}} +} diff --git a/vendor/google.golang.org/grpc/balancer/balancer.go b/vendor/google.golang.org/grpc/balancer/balancer.go new file mode 100644 index 00000000000..63b8d71371e --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer/balancer.go @@ -0,0 +1,228 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package balancer defines APIs for load balancing in gRPC. +// All APIs in this package are experimental. +package balancer + +import ( + "errors" + "net" + "strings" + + "golang.org/x/net/context" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/resolver" +) + +var ( + // m is a map from name to balancer builder. + m = make(map[string]Builder) +) + +// Register registers the balancer builder to the balancer map. b.Name +// (lowercased) will be used as the name registered with this builder. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Balancers are +// registered with the same name, the one registered last will take effect. +func Register(b Builder) { + m[strings.ToLower(b.Name())] = b +} + +// Get returns the resolver builder registered with the given name. +// Note that the compare is done in a case-insenstive fashion. +// If no builder is register with the name, nil will be returned. +func Get(name string) Builder { + if b, ok := m[strings.ToLower(name)]; ok { + return b + } + return nil +} + +// SubConn represents a gRPC sub connection. +// Each sub connection contains a list of addresses. gRPC will +// try to connect to them (in sequence), and stop trying the +// remainder once one connection is successful. +// +// The reconnect backoff will be applied on the list, not a single address. +// For example, try_on_all_addresses -> backoff -> try_on_all_addresses. +// +// All SubConns start in IDLE, and will not try to connect. To trigger +// the connecting, Balancers must call Connect. +// When the connection encounters an error, it will reconnect immediately. +// When the connection becomes IDLE, it will not reconnect unless Connect is +// called. +// +// This interface is to be implemented by gRPC. Users should not need a +// brand new implementation of this interface. For the situations like +// testing, the new implementation should embed this interface. This allows +// gRPC to add new methods to this interface. +type SubConn interface { + // UpdateAddresses updates the addresses used in this SubConn. + // gRPC checks if currently-connected address is still in the new list. + // If it's in the list, the connection will be kept. + // If it's not in the list, the connection will gracefully closed, and + // a new connection will be created. + // + // This will trigger a state transition for the SubConn. + UpdateAddresses([]resolver.Address) + // Connect starts the connecting for this SubConn. + Connect() +} + +// NewSubConnOptions contains options to create new SubConn. +type NewSubConnOptions struct{} + +// ClientConn represents a gRPC ClientConn. +// +// This interface is to be implemented by gRPC. Users should not need a +// brand new implementation of this interface. For the situations like +// testing, the new implementation should embed this interface. This allows +// gRPC to add new methods to this interface. +type ClientConn interface { + // NewSubConn is called by balancer to create a new SubConn. + // It doesn't block and wait for the connections to be established. + // Behaviors of the SubConn can be controlled by options. + NewSubConn([]resolver.Address, NewSubConnOptions) (SubConn, error) + // RemoveSubConn removes the SubConn from ClientConn. + // The SubConn will be shutdown. + RemoveSubConn(SubConn) + + // UpdateBalancerState is called by balancer to nofity gRPC that some internal + // state in balancer has changed. + // + // gRPC will update the connectivity state of the ClientConn, and will call pick + // on the new picker to pick new SubConn. + UpdateBalancerState(s connectivity.State, p Picker) + + // ResolveNow is called by balancer to notify gRPC to do a name resolving. + ResolveNow(resolver.ResolveNowOption) + + // Target returns the dial target for this ClientConn. + Target() string +} + +// BuildOptions contains additional information for Build. +type BuildOptions struct { + // DialCreds is the transport credential the Balancer implementation can + // use to dial to a remote load balancer server. The Balancer implementations + // can ignore this if it does not need to talk to another party securely. + DialCreds credentials.TransportCredentials + // Dialer is the custom dialer the Balancer implementation can use to dial + // to a remote load balancer server. The Balancer implementations + // can ignore this if it doesn't need to talk to remote balancer. + Dialer func(context.Context, string) (net.Conn, error) + // ChannelzParentID is the entity parent's channelz unique identification number. + ChannelzParentID int64 +} + +// Builder creates a balancer. +type Builder interface { + // Build creates a new balancer with the ClientConn. + Build(cc ClientConn, opts BuildOptions) Balancer + // Name returns the name of balancers built by this builder. + // It will be used to pick balancers (for example in service config). + Name() string +} + +// PickOptions contains addition information for the Pick operation. +type PickOptions struct{} + +// DoneInfo contains additional information for done. +type DoneInfo struct { + // Err is the rpc error the RPC finished with. It could be nil. + Err error + // BytesSent indicates if any bytes have been sent to the server. + BytesSent bool + // BytesReceived indicates if any byte has been received from the server. + BytesReceived bool +} + +var ( + // ErrNoSubConnAvailable indicates no SubConn is available for pick(). + // gRPC will block the RPC until a new picker is available via UpdateBalancerState(). + ErrNoSubConnAvailable = errors.New("no SubConn is available") + // ErrTransientFailure indicates all SubConns are in TransientFailure. + // WaitForReady RPCs will block, non-WaitForReady RPCs will fail. + ErrTransientFailure = errors.New("all SubConns are in TransientFailure") +) + +// Picker is used by gRPC to pick a SubConn to send an RPC. +// Balancer is expected to generate a new picker from its snapshot every time its +// internal state has changed. +// +// The pickers used by gRPC can be updated by ClientConn.UpdateBalancerState(). +type Picker interface { + // Pick returns the SubConn to be used to send the RPC. + // The returned SubConn must be one returned by NewSubConn(). + // + // This functions is expected to return: + // - a SubConn that is known to be READY; + // - ErrNoSubConnAvailable if no SubConn is available, but progress is being + // made (for example, some SubConn is in CONNECTING mode); + // - other errors if no active connecting is happening (for example, all SubConn + // are in TRANSIENT_FAILURE mode). + // + // If a SubConn is returned: + // - If it is READY, gRPC will send the RPC on it; + // - If it is not ready, or becomes not ready after it's returned, gRPC will block + // until UpdateBalancerState() is called and will call pick on the new picker. + // + // If the returned error is not nil: + // - If the error is ErrNoSubConnAvailable, gRPC will block until UpdateBalancerState() + // - If the error is ErrTransientFailure: + // - If the RPC is wait-for-ready, gRPC will block until UpdateBalancerState() + // is called to pick again; + // - Otherwise, RPC will fail with unavailable error. + // - Else (error is other non-nil error): + // - The RPC will fail with unavailable error. + // + // The returned done() function will be called once the rpc has finished, with the + // final status of that RPC. + // done may be nil if balancer doesn't care about the RPC status. + Pick(ctx context.Context, opts PickOptions) (conn SubConn, done func(DoneInfo), err error) +} + +// Balancer takes input from gRPC, manages SubConns, and collects and aggregates +// the connectivity states. +// +// It also generates and updates the Picker used by gRPC to pick SubConns for RPCs. +// +// HandleSubConnectionStateChange, HandleResolvedAddrs and Close are guaranteed +// to be called synchronously from the same goroutine. +// There's no guarantee on picker.Pick, it may be called anytime. +type Balancer interface { + // HandleSubConnStateChange is called by gRPC when the connectivity state + // of sc has changed. + // Balancer is expected to aggregate all the state of SubConn and report + // that back to gRPC. + // Balancer should also generate and update Pickers when its internal state has + // been changed by the new state. + HandleSubConnStateChange(sc SubConn, state connectivity.State) + // HandleResolvedAddrs is called by gRPC to send updated resolved addresses to + // balancers. + // Balancer can create new SubConn or remove SubConn with the addresses. + // An empty address slice and a non-nil error will be passed if the resolver returns + // non-nil error to gRPC. + HandleResolvedAddrs([]resolver.Address, error) + // Close closes the balancer. The balancer is not required to call + // ClientConn.RemoveSubConn for its existing SubConns. + Close() +} diff --git a/vendor/google.golang.org/grpc/balancer_conn_wrappers.go b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go new file mode 100644 index 00000000000..c23f81706fb --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go @@ -0,0 +1,300 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" +) + +// scStateUpdate contains the subConn and the new state it changed to. +type scStateUpdate struct { + sc balancer.SubConn + state connectivity.State +} + +// scStateUpdateBuffer is an unbounded channel for scStateChangeTuple. +// TODO make a general purpose buffer that uses interface{}. +type scStateUpdateBuffer struct { + c chan *scStateUpdate + mu sync.Mutex + backlog []*scStateUpdate +} + +func newSCStateUpdateBuffer() *scStateUpdateBuffer { + return &scStateUpdateBuffer{ + c: make(chan *scStateUpdate, 1), + } +} + +func (b *scStateUpdateBuffer) put(t *scStateUpdate) { + b.mu.Lock() + defer b.mu.Unlock() + if len(b.backlog) == 0 { + select { + case b.c <- t: + return + default: + } + } + b.backlog = append(b.backlog, t) +} + +func (b *scStateUpdateBuffer) load() { + b.mu.Lock() + defer b.mu.Unlock() + if len(b.backlog) > 0 { + select { + case b.c <- b.backlog[0]: + b.backlog[0] = nil + b.backlog = b.backlog[1:] + default: + } + } +} + +// get returns the channel that the scStateUpdate will be sent to. +// +// Upon receiving, the caller should call load to send another +// scStateChangeTuple onto the channel if there is any. +func (b *scStateUpdateBuffer) get() <-chan *scStateUpdate { + return b.c +} + +// resolverUpdate contains the new resolved addresses or error if there's +// any. +type resolverUpdate struct { + addrs []resolver.Address + err error +} + +// ccBalancerWrapper is a wrapper on top of cc for balancers. +// It implements balancer.ClientConn interface. +type ccBalancerWrapper struct { + cc *ClientConn + balancer balancer.Balancer + stateChangeQueue *scStateUpdateBuffer + resolverUpdateCh chan *resolverUpdate + done chan struct{} + + mu sync.Mutex + subConns map[*acBalancerWrapper]struct{} +} + +func newCCBalancerWrapper(cc *ClientConn, b balancer.Builder, bopts balancer.BuildOptions) *ccBalancerWrapper { + ccb := &ccBalancerWrapper{ + cc: cc, + stateChangeQueue: newSCStateUpdateBuffer(), + resolverUpdateCh: make(chan *resolverUpdate, 1), + done: make(chan struct{}), + subConns: make(map[*acBalancerWrapper]struct{}), + } + go ccb.watcher() + ccb.balancer = b.Build(ccb, bopts) + return ccb +} + +// watcher balancer functions sequentially, so the balancer can be implemented +// lock-free. +func (ccb *ccBalancerWrapper) watcher() { + for { + select { + case t := <-ccb.stateChangeQueue.get(): + ccb.stateChangeQueue.load() + select { + case <-ccb.done: + ccb.balancer.Close() + return + default: + } + ccb.balancer.HandleSubConnStateChange(t.sc, t.state) + case t := <-ccb.resolverUpdateCh: + select { + case <-ccb.done: + ccb.balancer.Close() + return + default: + } + ccb.balancer.HandleResolvedAddrs(t.addrs, t.err) + case <-ccb.done: + } + + select { + case <-ccb.done: + ccb.balancer.Close() + ccb.mu.Lock() + scs := ccb.subConns + ccb.subConns = nil + ccb.mu.Unlock() + for acbw := range scs { + ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain) + } + return + default: + } + } +} + +func (ccb *ccBalancerWrapper) close() { + close(ccb.done) +} + +func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { + // When updating addresses for a SubConn, if the address in use is not in + // the new addresses, the old ac will be tearDown() and a new ac will be + // created. tearDown() generates a state change with Shutdown state, we + // don't want the balancer to receive this state change. So before + // tearDown() on the old ac, ac.acbw (acWrapper) will be set to nil, and + // this function will be called with (nil, Shutdown). We don't need to call + // balancer method in this case. + if sc == nil { + return + } + ccb.stateChangeQueue.put(&scStateUpdate{ + sc: sc, + state: s, + }) +} + +func (ccb *ccBalancerWrapper) handleResolvedAddrs(addrs []resolver.Address, err error) { + select { + case <-ccb.resolverUpdateCh: + default: + } + ccb.resolverUpdateCh <- &resolverUpdate{ + addrs: addrs, + err: err, + } +} + +func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + if len(addrs) <= 0 { + return nil, fmt.Errorf("grpc: cannot create SubConn with empty address list") + } + ccb.mu.Lock() + defer ccb.mu.Unlock() + if ccb.subConns == nil { + return nil, fmt.Errorf("grpc: ClientConn balancer wrapper was closed") + } + ac, err := ccb.cc.newAddrConn(addrs) + if err != nil { + return nil, err + } + acbw := &acBalancerWrapper{ac: ac} + acbw.ac.mu.Lock() + ac.acbw = acbw + acbw.ac.mu.Unlock() + ccb.subConns[acbw] = struct{}{} + return acbw, nil +} + +func (ccb *ccBalancerWrapper) RemoveSubConn(sc balancer.SubConn) { + acbw, ok := sc.(*acBalancerWrapper) + if !ok { + return + } + ccb.mu.Lock() + defer ccb.mu.Unlock() + if ccb.subConns == nil { + return + } + delete(ccb.subConns, acbw) + ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain) +} + +func (ccb *ccBalancerWrapper) UpdateBalancerState(s connectivity.State, p balancer.Picker) { + ccb.mu.Lock() + defer ccb.mu.Unlock() + if ccb.subConns == nil { + return + } + ccb.cc.csMgr.updateState(s) + ccb.cc.blockingpicker.updatePicker(p) +} + +func (ccb *ccBalancerWrapper) ResolveNow(o resolver.ResolveNowOption) { + ccb.cc.resolveNow(o) +} + +func (ccb *ccBalancerWrapper) Target() string { + return ccb.cc.target +} + +// acBalancerWrapper is a wrapper on top of ac for balancers. +// It implements balancer.SubConn interface. +type acBalancerWrapper struct { + mu sync.Mutex + ac *addrConn +} + +func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) { + acbw.mu.Lock() + defer acbw.mu.Unlock() + if len(addrs) <= 0 { + acbw.ac.tearDown(errConnDrain) + return + } + if !acbw.ac.tryUpdateAddrs(addrs) { + cc := acbw.ac.cc + acbw.ac.mu.Lock() + // Set old ac.acbw to nil so the Shutdown state update will be ignored + // by balancer. + // + // TODO(bar) the state transition could be wrong when tearDown() old ac + // and creating new ac, fix the transition. + acbw.ac.acbw = nil + acbw.ac.mu.Unlock() + acState := acbw.ac.getState() + acbw.ac.tearDown(errConnDrain) + + if acState == connectivity.Shutdown { + return + } + + ac, err := cc.newAddrConn(addrs) + if err != nil { + grpclog.Warningf("acBalancerWrapper: UpdateAddresses: failed to newAddrConn: %v", err) + return + } + acbw.ac = ac + ac.mu.Lock() + ac.acbw = acbw + ac.mu.Unlock() + if acState != connectivity.Idle { + ac.connect() + } + } +} + +func (acbw *acBalancerWrapper) Connect() { + acbw.mu.Lock() + defer acbw.mu.Unlock() + acbw.ac.connect() +} + +func (acbw *acBalancerWrapper) getAddrConn() *addrConn { + acbw.mu.Lock() + defer acbw.mu.Unlock() + return acbw.ac +} diff --git a/vendor/google.golang.org/grpc/balancer_v1_wrapper.go b/vendor/google.golang.org/grpc/balancer_v1_wrapper.go new file mode 100644 index 00000000000..b7abc6b7457 --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer_v1_wrapper.go @@ -0,0 +1,372 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "strings" + "sync" + + "golang.org/x/net/context" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" +) + +type balancerWrapperBuilder struct { + b Balancer // The v1 balancer. +} + +func (bwb *balancerWrapperBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + targetAddr := cc.Target() + targetSplitted := strings.Split(targetAddr, ":///") + if len(targetSplitted) >= 2 { + targetAddr = targetSplitted[1] + } + + bwb.b.Start(targetAddr, BalancerConfig{ + DialCreds: opts.DialCreds, + Dialer: opts.Dialer, + }) + _, pickfirst := bwb.b.(*pickFirst) + bw := &balancerWrapper{ + balancer: bwb.b, + pickfirst: pickfirst, + cc: cc, + targetAddr: targetAddr, + startCh: make(chan struct{}), + conns: make(map[resolver.Address]balancer.SubConn), + connSt: make(map[balancer.SubConn]*scState), + csEvltr: &connectivityStateEvaluator{}, + state: connectivity.Idle, + } + cc.UpdateBalancerState(connectivity.Idle, bw) + go bw.lbWatcher() + return bw +} + +func (bwb *balancerWrapperBuilder) Name() string { + return "wrapper" +} + +type scState struct { + addr Address // The v1 address type. + s connectivity.State + down func(error) +} + +type balancerWrapper struct { + balancer Balancer // The v1 balancer. + pickfirst bool + + cc balancer.ClientConn + targetAddr string // Target without the scheme. + + // To aggregate the connectivity state. + csEvltr *connectivityStateEvaluator + state connectivity.State + + mu sync.Mutex + conns map[resolver.Address]balancer.SubConn + connSt map[balancer.SubConn]*scState + // This channel is closed when handling the first resolver result. + // lbWatcher blocks until this is closed, to avoid race between + // - NewSubConn is created, cc wants to notify balancer of state changes; + // - Build hasn't return, cc doesn't have access to balancer. + startCh chan struct{} +} + +// lbWatcher watches the Notify channel of the balancer and manages +// connections accordingly. +func (bw *balancerWrapper) lbWatcher() { + <-bw.startCh + notifyCh := bw.balancer.Notify() + if notifyCh == nil { + // There's no resolver in the balancer. Connect directly. + a := resolver.Address{ + Addr: bw.targetAddr, + Type: resolver.Backend, + } + sc, err := bw.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{}) + if err != nil { + grpclog.Warningf("Error creating connection to %v. Err: %v", a, err) + } else { + bw.mu.Lock() + bw.conns[a] = sc + bw.connSt[sc] = &scState{ + addr: Address{Addr: bw.targetAddr}, + s: connectivity.Idle, + } + bw.mu.Unlock() + sc.Connect() + } + return + } + + for addrs := range notifyCh { + grpclog.Infof("balancerWrapper: got update addr from Notify: %v\n", addrs) + if bw.pickfirst { + var ( + oldA resolver.Address + oldSC balancer.SubConn + ) + bw.mu.Lock() + for oldA, oldSC = range bw.conns { + break + } + bw.mu.Unlock() + if len(addrs) <= 0 { + if oldSC != nil { + // Teardown old sc. + bw.mu.Lock() + delete(bw.conns, oldA) + delete(bw.connSt, oldSC) + bw.mu.Unlock() + bw.cc.RemoveSubConn(oldSC) + } + continue + } + + var newAddrs []resolver.Address + for _, a := range addrs { + newAddr := resolver.Address{ + Addr: a.Addr, + Type: resolver.Backend, // All addresses from balancer are all backends. + ServerName: "", + Metadata: a.Metadata, + } + newAddrs = append(newAddrs, newAddr) + } + if oldSC == nil { + // Create new sc. + sc, err := bw.cc.NewSubConn(newAddrs, balancer.NewSubConnOptions{}) + if err != nil { + grpclog.Warningf("Error creating connection to %v. Err: %v", newAddrs, err) + } else { + bw.mu.Lock() + // For pickfirst, there should be only one SubConn, so the + // address doesn't matter. All states updating (up and down) + // and picking should all happen on that only SubConn. + bw.conns[resolver.Address{}] = sc + bw.connSt[sc] = &scState{ + addr: addrs[0], // Use the first address. + s: connectivity.Idle, + } + bw.mu.Unlock() + sc.Connect() + } + } else { + bw.mu.Lock() + bw.connSt[oldSC].addr = addrs[0] + bw.mu.Unlock() + oldSC.UpdateAddresses(newAddrs) + } + } else { + var ( + add []resolver.Address // Addresses need to setup connections. + del []balancer.SubConn // Connections need to tear down. + ) + resAddrs := make(map[resolver.Address]Address) + for _, a := range addrs { + resAddrs[resolver.Address{ + Addr: a.Addr, + Type: resolver.Backend, // All addresses from balancer are all backends. + ServerName: "", + Metadata: a.Metadata, + }] = a + } + bw.mu.Lock() + for a := range resAddrs { + if _, ok := bw.conns[a]; !ok { + add = append(add, a) + } + } + for a, c := range bw.conns { + if _, ok := resAddrs[a]; !ok { + del = append(del, c) + delete(bw.conns, a) + // Keep the state of this sc in bw.connSt until its state becomes Shutdown. + } + } + bw.mu.Unlock() + for _, a := range add { + sc, err := bw.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{}) + if err != nil { + grpclog.Warningf("Error creating connection to %v. Err: %v", a, err) + } else { + bw.mu.Lock() + bw.conns[a] = sc + bw.connSt[sc] = &scState{ + addr: resAddrs[a], + s: connectivity.Idle, + } + bw.mu.Unlock() + sc.Connect() + } + } + for _, c := range del { + bw.cc.RemoveSubConn(c) + } + } + } +} + +func (bw *balancerWrapper) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { + bw.mu.Lock() + defer bw.mu.Unlock() + scSt, ok := bw.connSt[sc] + if !ok { + return + } + if s == connectivity.Idle { + sc.Connect() + } + oldS := scSt.s + scSt.s = s + if oldS != connectivity.Ready && s == connectivity.Ready { + scSt.down = bw.balancer.Up(scSt.addr) + } else if oldS == connectivity.Ready && s != connectivity.Ready { + if scSt.down != nil { + scSt.down(errConnClosing) + } + } + sa := bw.csEvltr.recordTransition(oldS, s) + if bw.state != sa { + bw.state = sa + } + bw.cc.UpdateBalancerState(bw.state, bw) + if s == connectivity.Shutdown { + // Remove state for this sc. + delete(bw.connSt, sc) + } +} + +func (bw *balancerWrapper) HandleResolvedAddrs([]resolver.Address, error) { + bw.mu.Lock() + defer bw.mu.Unlock() + select { + case <-bw.startCh: + default: + close(bw.startCh) + } + // There should be a resolver inside the balancer. + // All updates here, if any, are ignored. +} + +func (bw *balancerWrapper) Close() { + bw.mu.Lock() + defer bw.mu.Unlock() + select { + case <-bw.startCh: + default: + close(bw.startCh) + } + bw.balancer.Close() +} + +// The picker is the balancerWrapper itself. +// Pick should never return ErrNoSubConnAvailable. +// It either blocks or returns error, consistent with v1 balancer Get(). +func (bw *balancerWrapper) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { + failfast := true // Default failfast is true. + if ss, ok := rpcInfoFromContext(ctx); ok { + failfast = ss.failfast + } + a, p, err := bw.balancer.Get(ctx, BalancerGetOptions{BlockingWait: !failfast}) + if err != nil { + return nil, nil, err + } + var done func(balancer.DoneInfo) + if p != nil { + done = func(i balancer.DoneInfo) { p() } + } + var sc balancer.SubConn + bw.mu.Lock() + defer bw.mu.Unlock() + if bw.pickfirst { + // Get the first sc in conns. + for _, sc = range bw.conns { + break + } + } else { + var ok bool + sc, ok = bw.conns[resolver.Address{ + Addr: a.Addr, + Type: resolver.Backend, + ServerName: "", + Metadata: a.Metadata, + }] + if !ok && failfast { + return nil, nil, status.Errorf(codes.Unavailable, "there is no connection available") + } + if s, ok := bw.connSt[sc]; failfast && (!ok || s.s != connectivity.Ready) { + // If the returned sc is not ready and RPC is failfast, + // return error, and this RPC will fail. + return nil, nil, status.Errorf(codes.Unavailable, "there is no connection available") + } + } + + return sc, done, nil +} + +// connectivityStateEvaluator gets updated by addrConns when their +// states transition, based on which it evaluates the state of +// ClientConn. +type connectivityStateEvaluator struct { + mu sync.Mutex + numReady uint64 // Number of addrConns in ready state. + numConnecting uint64 // Number of addrConns in connecting state. + numTransientFailure uint64 // Number of addrConns in transientFailure. +} + +// recordTransition records state change happening in every subConn and based on +// that it evaluates what aggregated state should be. +// It can only transition between Ready, Connecting and TransientFailure. Other states, +// Idle and Shutdown are transitioned into by ClientConn; in the beginning of the connection +// before any subConn is created ClientConn is in idle state. In the end when ClientConn +// closes it is in Shutdown state. +// TODO Note that in later releases, a ClientConn with no activity will be put into an Idle state. +func (cse *connectivityStateEvaluator) recordTransition(oldState, newState connectivity.State) connectivity.State { + cse.mu.Lock() + defer cse.mu.Unlock() + + // Update counters. + for idx, state := range []connectivity.State{oldState, newState} { + updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. + switch state { + case connectivity.Ready: + cse.numReady += updateVal + case connectivity.Connecting: + cse.numConnecting += updateVal + case connectivity.TransientFailure: + cse.numTransientFailure += updateVal + } + } + + // Evaluate. + if cse.numReady > 0 { + return connectivity.Ready + } + if cse.numConnecting > 0 { + return connectivity.Connecting + } + return connectivity.TransientFailure +} diff --git a/vendor/google.golang.org/grpc/call.go b/vendor/google.golang.org/grpc/call.go index fc8e18afc29..f73b7d5528f 100644 --- a/vendor/google.golang.org/grpc/call.go +++ b/vendor/google.golang.org/grpc/call.go @@ -1,276 +1,93 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ package grpc import ( - "bytes" - "io" - "math" - "time" - "golang.org/x/net/context" - "golang.org/x/net/trace" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/stats" - "google.golang.org/grpc/transport" ) -// recvResponse receives and parses an RPC response. -// On error, it returns the error and indicates whether the call should be retried. +// Invoke sends the RPC request on the wire and returns after response is +// received. This is typically called by generated code. // -// TODO(zhaoq): Check whether the received message sequence is valid. -// TODO ctx is used for stats collection and processing. It is the context passed from the application. -func recvResponse(ctx context.Context, dopts dialOptions, t transport.ClientTransport, c *callInfo, stream *transport.Stream, reply interface{}) (err error) { - // Try to acquire header metadata from the server if there is any. - defer func() { - if err != nil { - if _, ok := err.(transport.ConnectionError); !ok { - t.CloseStream(stream, err) - } - } - }() - c.headerMD, err = stream.Header() - if err != nil { - return - } - p := &parser{r: stream} - var inPayload *stats.InPayload - if stats.On() { - inPayload = &stats.InPayload{ - Client: true, - } - } - for { - if err = recv(p, dopts.codec, stream, dopts.dc, reply, math.MaxInt32, inPayload); err != nil { - if err == io.EOF { - break - } - return - } - } - if inPayload != nil && err == io.EOF && stream.StatusCode() == codes.OK { - // TODO in the current implementation, inTrailer may be handled before inPayload in some cases. - // Fix the order if necessary. - stats.HandleRPC(ctx, inPayload) +// All errors returned by Invoke are compatible with the status package. +func (cc *ClientConn) Invoke(ctx context.Context, method string, args, reply interface{}, opts ...CallOption) error { + // allow interceptor to see all applicable call options, which means those + // configured as defaults from dial option as well as per-call options + opts = combine(cc.dopts.callOptions, opts) + + if cc.dopts.unaryInt != nil { + return cc.dopts.unaryInt(ctx, method, args, reply, cc, invoke, opts...) } - c.trailerMD = stream.Trailer() - return nil + return invoke(ctx, method, args, reply, cc, opts...) } -// sendRequest writes out various information of an RPC such as Context and Message. -func sendRequest(ctx context.Context, codec Codec, compressor Compressor, callHdr *transport.CallHdr, t transport.ClientTransport, args interface{}, opts *transport.Options) (_ *transport.Stream, err error) { - stream, err := t.NewStream(ctx, callHdr) - if err != nil { - return nil, err - } - defer func() { - if err != nil { - // If err is connection error, t will be closed, no need to close stream here. - if _, ok := err.(transport.ConnectionError); !ok { - t.CloseStream(stream, err) - } - } - }() - var ( - cbuf *bytes.Buffer - outPayload *stats.OutPayload - ) - if compressor != nil { - cbuf = new(bytes.Buffer) - } - if stats.On() { - outPayload = &stats.OutPayload{ - Client: true, - } - } - outBuf, err := encode(codec, args, compressor, cbuf, outPayload) - if err != nil { - return nil, Errorf(codes.Internal, "grpc: %v", err) - } - err = t.Write(stream, outBuf, opts) - if err == nil && outPayload != nil { - outPayload.SentTime = time.Now() - stats.HandleRPC(ctx, outPayload) - } - // t.NewStream(...) could lead to an early rejection of the RPC (e.g., the service/method - // does not exist.) so that t.Write could get io.EOF from wait(...). Leave the following - // recvResponse to get the final status. - if err != nil && err != io.EOF { - return nil, err - } - // Sent successfully. - return stream, nil +func combine(o1 []CallOption, o2 []CallOption) []CallOption { + // we don't use append because o1 could have extra capacity whose + // elements would be overwritten, which could cause inadvertent + // sharing (and race connditions) between concurrent calls + if len(o1) == 0 { + return o2 + } else if len(o2) == 0 { + return o1 + } + ret := make([]CallOption, len(o1)+len(o2)) + copy(ret, o1) + copy(ret[len(o1):], o2) + return ret } -// Invoke sends the RPC request on the wire and returns after response is received. -// Invoke is called by generated code. Also users can call Invoke directly when it -// is really needed in their use cases. +// Invoke sends the RPC request on the wire and returns after response is +// received. This is typically called by generated code. +// +// DEPRECATED: Use ClientConn.Invoke instead. func Invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) error { - if cc.dopts.unaryInt != nil { - return cc.dopts.unaryInt(ctx, method, args, reply, cc, invoke, opts...) - } - return invoke(ctx, method, args, reply, cc, opts...) + return cc.Invoke(ctx, method, args, reply, opts...) } -func invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) (e error) { - c := defaultCallInfo - for _, o := range opts { - if err := o.before(&c); err != nil { - return toRPCErr(err) - } - } - defer func() { - for _, o := range opts { - o.after(&c) - } - }() - if EnableTracing { - c.traceInfo.tr = trace.New("grpc.Sent."+methodFamily(method), method) - defer c.traceInfo.tr.Finish() - c.traceInfo.firstLine.client = true - if deadline, ok := ctx.Deadline(); ok { - c.traceInfo.firstLine.deadline = deadline.Sub(time.Now()) - } - c.traceInfo.tr.LazyLog(&c.traceInfo.firstLine, false) - // TODO(dsymonds): Arrange for c.traceInfo.firstLine.remoteAddr to be set. - defer func() { - if e != nil { - c.traceInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{e}}, true) - c.traceInfo.tr.SetError() - } - }() - } - if stats.On() { - ctx = stats.TagRPC(ctx, &stats.RPCTagInfo{FullMethodName: method}) - begin := &stats.Begin{ - Client: true, - BeginTime: time.Now(), - FailFast: c.failFast, - } - stats.HandleRPC(ctx, begin) - } - defer func() { - if stats.On() { - end := &stats.End{ - Client: true, - EndTime: time.Now(), - Error: e, - } - stats.HandleRPC(ctx, end) - } - }() - topts := &transport.Options{ - Last: true, - Delay: false, - } +var unaryStreamDesc = &StreamDesc{ServerStreams: false, ClientStreams: false} + +func invoke(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error { + // TODO: implement retries in clientStream and make this simply + // newClientStream, SendMsg, RecvMsg. + firstAttempt := true for { - var ( - err error - t transport.ClientTransport - stream *transport.Stream - // Record the put handler from Balancer.Get(...). It is called once the - // RPC has completed or failed. - put func() - ) - // TODO(zhaoq): Need a formal spec of fail-fast. - callHdr := &transport.CallHdr{ - Host: cc.authority, - Method: method, - } - if cc.dopts.cp != nil { - callHdr.SendCompress = cc.dopts.cp.Type() - } - gopts := BalancerGetOptions{ - BlockingWait: !c.failFast, - } - t, put, err = cc.getTransport(ctx, gopts) + csInt, err := newClientStream(ctx, unaryStreamDesc, cc, method, opts...) if err != nil { - // TODO(zhaoq): Probably revisit the error handling. - if _, ok := err.(*rpcError); ok { - return err - } - if err == errConnClosing || err == errConnUnavailable { - if c.failFast { - return Errorf(codes.Unavailable, "%v", err) - } - continue - } - // All the other errors are treated as Internal errors. - return Errorf(codes.Internal, "%v", err) - } - if c.traceInfo.tr != nil { - c.traceInfo.tr.LazyLog(&payload{sent: true, msg: args}, true) + return err } - stream, err = sendRequest(ctx, cc.dopts.codec, cc.dopts.cp, callHdr, t, args, topts) - if err != nil { - if put != nil { - put() - put = nil - } - // Retry a non-failfast RPC when - // i) there is a connection error; or - // ii) the server started to drain before this RPC was initiated. - if _, ok := err.(transport.ConnectionError); ok || err == transport.ErrStreamDrain { - if c.failFast { - return toRPCErr(err) - } + cs := csInt.(*clientStream) + if err := cs.SendMsg(req); err != nil { + if !cs.c.failFast && cs.attempt.s.Unprocessed() && firstAttempt { + // TODO: Add a field to header for grpc-transparent-retry-attempts + firstAttempt = false continue } - return toRPCErr(err) + return err } - err = recvResponse(ctx, cc.dopts, t, &c, stream, reply) - if err != nil { - if put != nil { - put() - put = nil - } - if _, ok := err.(transport.ConnectionError); ok || err == transport.ErrStreamDrain { - if c.failFast { - return toRPCErr(err) - } + if err := cs.RecvMsg(reply); err != nil { + if !cs.c.failFast && cs.attempt.s.Unprocessed() && firstAttempt { + // TODO: Add a field to header for grpc-transparent-retry-attempts + firstAttempt = false continue } - return toRPCErr(err) - } - if c.traceInfo.tr != nil { - c.traceInfo.tr.LazyLog(&payload{sent: false, msg: reply}, true) - } - t.CloseStream(stream, nil) - if put != nil { - put() - put = nil + return err } - return Errorf(stream.StatusCode(), "%s", stream.StatusDesc()) + return nil } } diff --git a/vendor/google.golang.org/grpc/channelz/funcs.go b/vendor/google.golang.org/grpc/channelz/funcs.go new file mode 100644 index 00000000000..586a0336b47 --- /dev/null +++ b/vendor/google.golang.org/grpc/channelz/funcs.go @@ -0,0 +1,573 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package channelz defines APIs for enabling channelz service, entry +// registration/deletion, and accessing channelz data. It also defines channelz +// metric struct formats. +// +// All APIs in this package are experimental. +package channelz + +import ( + "sort" + "sync" + "sync/atomic" + + "google.golang.org/grpc/grpclog" +) + +var ( + db dbWrapper + idGen idGenerator + // EntryPerPage defines the number of channelz entries to be shown on a web page. + EntryPerPage = 50 + curState int32 +) + +// TurnOn turns on channelz data collection. +func TurnOn() { + if !IsOn() { + NewChannelzStorage() + atomic.StoreInt32(&curState, 1) + } +} + +// IsOn returns whether channelz data collection is on. +func IsOn() bool { + return atomic.CompareAndSwapInt32(&curState, 1, 1) +} + +// dbWarpper wraps around a reference to internal channelz data storage, and +// provide synchronized functionality to set and get the reference. +type dbWrapper struct { + mu sync.RWMutex + DB *channelMap +} + +func (d *dbWrapper) set(db *channelMap) { + d.mu.Lock() + d.DB = db + d.mu.Unlock() +} + +func (d *dbWrapper) get() *channelMap { + d.mu.RLock() + defer d.mu.RUnlock() + return d.DB +} + +// NewChannelzStorage initializes channelz data storage and id generator. +// +// Note: This function is exported for testing purpose only. User should not call +// it in most cases. +func NewChannelzStorage() { + db.set(&channelMap{ + topLevelChannels: make(map[int64]struct{}), + channels: make(map[int64]*channel), + listenSockets: make(map[int64]*listenSocket), + normalSockets: make(map[int64]*normalSocket), + servers: make(map[int64]*server), + subChannels: make(map[int64]*subChannel), + }) + idGen.reset() +} + +// GetTopChannels returns a slice of top channel's ChannelMetric, along with a +// boolean indicating whether there's more top channels to be queried for. +// +// The arg id specifies that only top channel with id at or above it will be included +// in the result. The returned slice is up to a length of EntryPerPage, and is +// sorted in ascending id order. +func GetTopChannels(id int64) ([]*ChannelMetric, bool) { + return db.get().GetTopChannels(id) +} + +// GetServers returns a slice of server's ServerMetric, along with a +// boolean indicating whether there's more servers to be queried for. +// +// The arg id specifies that only server with id at or above it will be included +// in the result. The returned slice is up to a length of EntryPerPage, and is +// sorted in ascending id order. +func GetServers(id int64) ([]*ServerMetric, bool) { + return db.get().GetServers(id) +} + +// GetServerSockets returns a slice of server's (identified by id) normal socket's +// SocketMetric, along with a boolean indicating whether there's more sockets to +// be queried for. +// +// The arg startID specifies that only sockets with id at or above it will be +// included in the result. The returned slice is up to a length of EntryPerPage, +// and is sorted in ascending id order. +func GetServerSockets(id int64, startID int64) ([]*SocketMetric, bool) { + return db.get().GetServerSockets(id, startID) +} + +// GetChannel returns the ChannelMetric for the channel (identified by id). +func GetChannel(id int64) *ChannelMetric { + return db.get().GetChannel(id) +} + +// GetSubChannel returns the SubChannelMetric for the subchannel (identified by id). +func GetSubChannel(id int64) *SubChannelMetric { + return db.get().GetSubChannel(id) +} + +// GetSocket returns the SocketInternalMetric for the socket (identified by id). +func GetSocket(id int64) *SocketMetric { + return db.get().GetSocket(id) +} + +// RegisterChannel registers the given channel c in channelz database with ref +// as its reference name, and add it to the child list of its parent (identified +// by pid). pid = 0 means no parent. It returns the unique channelz tracking id +// assigned to this channel. +func RegisterChannel(c Channel, pid int64, ref string) int64 { + id := idGen.genID() + cn := &channel{ + refName: ref, + c: c, + subChans: make(map[int64]string), + nestedChans: make(map[int64]string), + id: id, + pid: pid, + } + if pid == 0 { + db.get().addChannel(id, cn, true, pid, ref) + } else { + db.get().addChannel(id, cn, false, pid, ref) + } + return id +} + +// RegisterSubChannel registers the given channel c in channelz database with ref +// as its reference name, and add it to the child list of its parent (identified +// by pid). It returns the unique channelz tracking id assigned to this subchannel. +func RegisterSubChannel(c Channel, pid int64, ref string) int64 { + if pid == 0 { + grpclog.Error("a SubChannel's parent id cannot be 0") + return 0 + } + id := idGen.genID() + sc := &subChannel{ + refName: ref, + c: c, + sockets: make(map[int64]string), + id: id, + pid: pid, + } + db.get().addSubChannel(id, sc, pid, ref) + return id +} + +// RegisterServer registers the given server s in channelz database. It returns +// the unique channelz tracking id assigned to this server. +func RegisterServer(s Server, ref string) int64 { + id := idGen.genID() + svr := &server{ + refName: ref, + s: s, + sockets: make(map[int64]string), + listenSockets: make(map[int64]string), + id: id, + } + db.get().addServer(id, svr) + return id +} + +// RegisterListenSocket registers the given listen socket s in channelz database +// with ref as its reference name, and add it to the child list of its parent +// (identified by pid). It returns the unique channelz tracking id assigned to +// this listen socket. +func RegisterListenSocket(s Socket, pid int64, ref string) int64 { + if pid == 0 { + grpclog.Error("a ListenSocket's parent id cannot be 0") + return 0 + } + id := idGen.genID() + ls := &listenSocket{refName: ref, s: s, id: id, pid: pid} + db.get().addListenSocket(id, ls, pid, ref) + return id +} + +// RegisterNormalSocket registers the given normal socket s in channelz database +// with ref as its reference name, and add it to the child list of its parent +// (identified by pid). It returns the unique channelz tracking id assigned to +// this normal socket. +func RegisterNormalSocket(s Socket, pid int64, ref string) int64 { + if pid == 0 { + grpclog.Error("a NormalSocket's parent id cannot be 0") + return 0 + } + id := idGen.genID() + ns := &normalSocket{refName: ref, s: s, id: id, pid: pid} + db.get().addNormalSocket(id, ns, pid, ref) + return id +} + +// RemoveEntry removes an entry with unique channelz trakcing id to be id from +// channelz database. +func RemoveEntry(id int64) { + db.get().removeEntry(id) +} + +// channelMap is the storage data structure for channelz. +// Methods of channelMap can be divided in two two categories with respect to locking. +// 1. Methods acquire the global lock. +// 2. Methods that can only be called when global lock is held. +// A second type of method need always to be called inside a first type of method. +type channelMap struct { + mu sync.RWMutex + topLevelChannels map[int64]struct{} + servers map[int64]*server + channels map[int64]*channel + subChannels map[int64]*subChannel + listenSockets map[int64]*listenSocket + normalSockets map[int64]*normalSocket +} + +func (c *channelMap) addServer(id int64, s *server) { + c.mu.Lock() + s.cm = c + c.servers[id] = s + c.mu.Unlock() +} + +func (c *channelMap) addChannel(id int64, cn *channel, isTopChannel bool, pid int64, ref string) { + c.mu.Lock() + cn.cm = c + c.channels[id] = cn + if isTopChannel { + c.topLevelChannels[id] = struct{}{} + } else { + c.findEntry(pid).addChild(id, cn) + } + c.mu.Unlock() +} + +func (c *channelMap) addSubChannel(id int64, sc *subChannel, pid int64, ref string) { + c.mu.Lock() + sc.cm = c + c.subChannels[id] = sc + c.findEntry(pid).addChild(id, sc) + c.mu.Unlock() +} + +func (c *channelMap) addListenSocket(id int64, ls *listenSocket, pid int64, ref string) { + c.mu.Lock() + ls.cm = c + c.listenSockets[id] = ls + c.findEntry(pid).addChild(id, ls) + c.mu.Unlock() +} + +func (c *channelMap) addNormalSocket(id int64, ns *normalSocket, pid int64, ref string) { + c.mu.Lock() + ns.cm = c + c.normalSockets[id] = ns + c.findEntry(pid).addChild(id, ns) + c.mu.Unlock() +} + +// removeEntry triggers the removal of an entry, which may not indeed delete the +// entry, if it has to wait on the deletion of its children, or may lead to a chain +// of entry deletion. For example, deleting the last socket of a gracefully shutting +// down server will lead to the server being also deleted. +func (c *channelMap) removeEntry(id int64) { + c.mu.Lock() + c.findEntry(id).triggerDelete() + c.mu.Unlock() +} + +// c.mu must be held by the caller. +func (c *channelMap) findEntry(id int64) entry { + var v entry + var ok bool + if v, ok = c.channels[id]; ok { + return v + } + if v, ok = c.subChannels[id]; ok { + return v + } + if v, ok = c.servers[id]; ok { + return v + } + if v, ok = c.listenSockets[id]; ok { + return v + } + if v, ok = c.normalSockets[id]; ok { + return v + } + return &dummyEntry{idNotFound: id} +} + +// c.mu must be held by the caller +// deleteEntry simply deletes an entry from the channelMap. Before calling this +// method, caller must check this entry is ready to be deleted, i.e removeEntry() +// has been called on it, and no children still exist. +// Conditionals are ordered by the expected frequency of deletion of each entity +// type, in order to optimize performance. +func (c *channelMap) deleteEntry(id int64) { + var ok bool + if _, ok = c.normalSockets[id]; ok { + delete(c.normalSockets, id) + return + } + if _, ok = c.subChannels[id]; ok { + delete(c.subChannels, id) + return + } + if _, ok = c.channels[id]; ok { + delete(c.channels, id) + delete(c.topLevelChannels, id) + return + } + if _, ok = c.listenSockets[id]; ok { + delete(c.listenSockets, id) + return + } + if _, ok = c.servers[id]; ok { + delete(c.servers, id) + return + } +} + +type int64Slice []int64 + +func (s int64Slice) Len() int { return len(s) } +func (s int64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s int64Slice) Less(i, j int) bool { return s[i] < s[j] } + +func copyMap(m map[int64]string) map[int64]string { + n := make(map[int64]string) + for k, v := range m { + n[k] = v + } + return n +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func (c *channelMap) GetTopChannels(id int64) ([]*ChannelMetric, bool) { + c.mu.RLock() + l := len(c.topLevelChannels) + ids := make([]int64, 0, l) + cns := make([]*channel, 0, min(l, EntryPerPage)) + + for k := range c.topLevelChannels { + ids = append(ids, k) + } + sort.Sort(int64Slice(ids)) + idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) + count := 0 + var end bool + var t []*ChannelMetric + for i, v := range ids[idx:] { + if count == EntryPerPage { + break + } + if cn, ok := c.channels[v]; ok { + cns = append(cns, cn) + t = append(t, &ChannelMetric{ + NestedChans: copyMap(cn.nestedChans), + SubChans: copyMap(cn.subChans), + }) + count++ + } + if i == len(ids[idx:])-1 { + end = true + break + } + } + c.mu.RUnlock() + if count == 0 { + end = true + } + + for i, cn := range cns { + t[i].ChannelData = cn.c.ChannelzMetric() + t[i].ID = cn.id + t[i].RefName = cn.refName + } + return t, end +} + +func (c *channelMap) GetServers(id int64) ([]*ServerMetric, bool) { + c.mu.RLock() + l := len(c.servers) + ids := make([]int64, 0, l) + ss := make([]*server, 0, min(l, EntryPerPage)) + for k := range c.servers { + ids = append(ids, k) + } + sort.Sort(int64Slice(ids)) + idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) + count := 0 + var end bool + var s []*ServerMetric + for i, v := range ids[idx:] { + if count == EntryPerPage { + break + } + if svr, ok := c.servers[v]; ok { + ss = append(ss, svr) + s = append(s, &ServerMetric{ + ListenSockets: copyMap(svr.listenSockets), + }) + count++ + } + if i == len(ids[idx:])-1 { + end = true + break + } + } + c.mu.RUnlock() + if count == 0 { + end = true + } + + for i, svr := range ss { + s[i].ServerData = svr.s.ChannelzMetric() + s[i].ID = svr.id + s[i].RefName = svr.refName + } + return s, end +} + +func (c *channelMap) GetServerSockets(id int64, startID int64) ([]*SocketMetric, bool) { + var svr *server + var ok bool + c.mu.RLock() + if svr, ok = c.servers[id]; !ok { + // server with id doesn't exist. + c.mu.RUnlock() + return nil, true + } + svrskts := svr.sockets + l := len(svrskts) + ids := make([]int64, 0, l) + sks := make([]*normalSocket, 0, min(l, EntryPerPage)) + for k := range svrskts { + ids = append(ids, k) + } + sort.Sort((int64Slice(ids))) + idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) + count := 0 + var end bool + for i, v := range ids[idx:] { + if count == EntryPerPage { + break + } + if ns, ok := c.normalSockets[v]; ok { + sks = append(sks, ns) + count++ + } + if i == len(ids[idx:])-1 { + end = true + break + } + } + c.mu.RUnlock() + if count == 0 { + end = true + } + var s []*SocketMetric + for _, ns := range sks { + sm := &SocketMetric{} + sm.SocketData = ns.s.ChannelzMetric() + sm.ID = ns.id + sm.RefName = ns.refName + s = append(s, sm) + } + return s, end +} + +func (c *channelMap) GetChannel(id int64) *ChannelMetric { + cm := &ChannelMetric{} + var cn *channel + var ok bool + c.mu.RLock() + if cn, ok = c.channels[id]; !ok { + // channel with id doesn't exist. + c.mu.RUnlock() + return nil + } + cm.NestedChans = copyMap(cn.nestedChans) + cm.SubChans = copyMap(cn.subChans) + c.mu.RUnlock() + cm.ChannelData = cn.c.ChannelzMetric() + cm.ID = cn.id + cm.RefName = cn.refName + return cm +} + +func (c *channelMap) GetSubChannel(id int64) *SubChannelMetric { + cm := &SubChannelMetric{} + var sc *subChannel + var ok bool + c.mu.RLock() + if sc, ok = c.subChannels[id]; !ok { + // subchannel with id doesn't exist. + c.mu.RUnlock() + return nil + } + cm.Sockets = copyMap(sc.sockets) + c.mu.RUnlock() + cm.ChannelData = sc.c.ChannelzMetric() + cm.ID = sc.id + cm.RefName = sc.refName + return cm +} + +func (c *channelMap) GetSocket(id int64) *SocketMetric { + sm := &SocketMetric{} + c.mu.RLock() + if ls, ok := c.listenSockets[id]; ok { + c.mu.RUnlock() + sm.SocketData = ls.s.ChannelzMetric() + sm.ID = ls.id + sm.RefName = ls.refName + return sm + } + if ns, ok := c.normalSockets[id]; ok { + c.mu.RUnlock() + sm.SocketData = ns.s.ChannelzMetric() + sm.ID = ns.id + sm.RefName = ns.refName + return sm + } + c.mu.RUnlock() + return nil +} + +type idGenerator struct { + id int64 +} + +func (i *idGenerator) reset() { + atomic.StoreInt64(&i.id, 0) +} + +func (i *idGenerator) genID() int64 { + return atomic.AddInt64(&i.id, 1) +} diff --git a/vendor/google.golang.org/grpc/channelz/types.go b/vendor/google.golang.org/grpc/channelz/types.go new file mode 100644 index 00000000000..153d75340e4 --- /dev/null +++ b/vendor/google.golang.org/grpc/channelz/types.go @@ -0,0 +1,418 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import ( + "net" + "time" + + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" +) + +// entry represents a node in the channelz database. +type entry interface { + // addChild adds a child e, whose channelz id is id to child list + addChild(id int64, e entry) + // deleteChild deletes a child with channelz id to be id from child list + deleteChild(id int64) + // triggerDelete tries to delete self from channelz database. However, if child + // list is not empty, then deletion from the database is on hold until the last + // child is deleted from database. + triggerDelete() + // deleteSelfIfReady check whether triggerDelete() has been called before, and whether child + // list is now empty. If both conditions are met, then delete self from database. + deleteSelfIfReady() +} + +// dummyEntry is a fake entry to handle entry not found case. +type dummyEntry struct { + idNotFound int64 +} + +func (d *dummyEntry) addChild(id int64, e entry) { + // Note: It is possible for a normal program to reach here under race condition. + // For example, there could be a race between ClientConn.Close() info being propagated + // to addrConn and http2Client. ClientConn.Close() cancel the context and result + // in http2Client to error. The error info is then caught by transport monitor + // and before addrConn.tearDown() is called in side ClientConn.Close(). Therefore, + // the addrConn will create a new transport. And when registering the new transport in + // channelz, its parent addrConn could have already been torn down and deleted + // from channelz tracking, and thus reach the code here. + grpclog.Infof("attempt to add child of type %T with id %d to a parent (id=%d) that doesn't currently exist", e, id, d.idNotFound) +} + +func (d *dummyEntry) deleteChild(id int64) { + // It is possible for a normal program to reach here under race condition. + // Refer to the example described in addChild(). + grpclog.Infof("attempt to delete child with id %d from a parent (id=%d) that doesn't currently exist", id, d.idNotFound) +} + +func (d *dummyEntry) triggerDelete() { + grpclog.Warningf("attempt to delete an entry (id=%d) that doesn't currently exist", d.idNotFound) +} + +func (*dummyEntry) deleteSelfIfReady() { + // code should not reach here. deleteSelfIfReady is always called on an existing entry. +} + +// ChannelMetric defines the info channelz provides for a specific Channel, which +// includes ChannelInternalMetric and channelz-specific data, such as channelz id, +// child list, etc. +type ChannelMetric struct { + // ID is the channelz id of this channel. + ID int64 + // RefName is the human readable reference string of this channel. + RefName string + // ChannelData contains channel internal metric reported by the channel through + // ChannelzMetric(). + ChannelData *ChannelInternalMetric + // NestedChans tracks the nested channel type children of this channel in the format of + // a map from nested channel channelz id to corresponding reference string. + NestedChans map[int64]string + // SubChans tracks the subchannel type children of this channel in the format of a + // map from subchannel channelz id to corresponding reference string. + SubChans map[int64]string + // Sockets tracks the socket type children of this channel in the format of a map + // from socket channelz id to corresponding reference string. + // Note current grpc implementation doesn't allow channel having sockets directly, + // therefore, this is field is unused. + Sockets map[int64]string +} + +// SubChannelMetric defines the info channelz provides for a specific SubChannel, +// which includes ChannelInternalMetric and channelz-specific data, such as +// channelz id, child list, etc. +type SubChannelMetric struct { + // ID is the channelz id of this subchannel. + ID int64 + // RefName is the human readable reference string of this subchannel. + RefName string + // ChannelData contains subchannel internal metric reported by the subchannel + // through ChannelzMetric(). + ChannelData *ChannelInternalMetric + // NestedChans tracks the nested channel type children of this subchannel in the format of + // a map from nested channel channelz id to corresponding reference string. + // Note current grpc implementation doesn't allow subchannel to have nested channels + // as children, therefore, this field is unused. + NestedChans map[int64]string + // SubChans tracks the subchannel type children of this subchannel in the format of a + // map from subchannel channelz id to corresponding reference string. + // Note current grpc implementation doesn't allow subchannel to have subchannels + // as children, therefore, this field is unused. + SubChans map[int64]string + // Sockets tracks the socket type children of this subchannel in the format of a map + // from socket channelz id to corresponding reference string. + Sockets map[int64]string +} + +// ChannelInternalMetric defines the struct that the implementor of Channel interface +// should return from ChannelzMetric(). +type ChannelInternalMetric struct { + // current connectivity state of the channel. + State connectivity.State + // The target this channel originally tried to connect to. May be absent + Target string + // The number of calls started on the channel. + CallsStarted int64 + // The number of calls that have completed with an OK status. + CallsSucceeded int64 + // The number of calls that have a completed with a non-OK status. + CallsFailed int64 + // The last time a call was started on the channel. + LastCallStartedTimestamp time.Time + //TODO: trace +} + +// Channel is the interface that should be satisfied in order to be tracked by +// channelz as Channel or SubChannel. +type Channel interface { + ChannelzMetric() *ChannelInternalMetric +} + +type channel struct { + refName string + c Channel + closeCalled bool + nestedChans map[int64]string + subChans map[int64]string + id int64 + pid int64 + cm *channelMap +} + +func (c *channel) addChild(id int64, e entry) { + switch v := e.(type) { + case *subChannel: + c.subChans[id] = v.refName + case *channel: + c.nestedChans[id] = v.refName + default: + grpclog.Errorf("cannot add a child (id = %d) of type %T to a channel", id, e) + } +} + +func (c *channel) deleteChild(id int64) { + delete(c.subChans, id) + delete(c.nestedChans, id) + c.deleteSelfIfReady() +} + +func (c *channel) triggerDelete() { + c.closeCalled = true + c.deleteSelfIfReady() +} + +func (c *channel) deleteSelfIfReady() { + if !c.closeCalled || len(c.subChans)+len(c.nestedChans) != 0 { + return + } + c.cm.deleteEntry(c.id) + // not top channel + if c.pid != 0 { + c.cm.findEntry(c.pid).deleteChild(c.id) + } +} + +type subChannel struct { + refName string + c Channel + closeCalled bool + sockets map[int64]string + id int64 + pid int64 + cm *channelMap +} + +func (sc *subChannel) addChild(id int64, e entry) { + if v, ok := e.(*normalSocket); ok { + sc.sockets[id] = v.refName + } else { + grpclog.Errorf("cannot add a child (id = %d) of type %T to a subChannel", id, e) + } +} + +func (sc *subChannel) deleteChild(id int64) { + delete(sc.sockets, id) + sc.deleteSelfIfReady() +} + +func (sc *subChannel) triggerDelete() { + sc.closeCalled = true + sc.deleteSelfIfReady() +} + +func (sc *subChannel) deleteSelfIfReady() { + if !sc.closeCalled || len(sc.sockets) != 0 { + return + } + sc.cm.deleteEntry(sc.id) + sc.cm.findEntry(sc.pid).deleteChild(sc.id) +} + +// SocketMetric defines the info channelz provides for a specific Socket, which +// includes SocketInternalMetric and channelz-specific data, such as channelz id, etc. +type SocketMetric struct { + // ID is the channelz id of this socket. + ID int64 + // RefName is the human readable reference string of this socket. + RefName string + // SocketData contains socket internal metric reported by the socket through + // ChannelzMetric(). + SocketData *SocketInternalMetric +} + +// SocketInternalMetric defines the struct that the implementor of Socket interface +// should return from ChannelzMetric(). +type SocketInternalMetric struct { + // The number of streams that have been started. + StreamsStarted int64 + // The number of streams that have ended successfully: + // On client side, receiving frame with eos bit set. + // On server side, sending frame with eos bit set. + StreamsSucceeded int64 + // The number of streams that have ended unsuccessfully: + // On client side, termination without receiving frame with eos bit set. + // On server side, termination without sending frame with eos bit set. + StreamsFailed int64 + // The number of messages successfully sent on this socket. + MessagesSent int64 + MessagesReceived int64 + // The number of keep alives sent. This is typically implemented with HTTP/2 + // ping messages. + KeepAlivesSent int64 + // The last time a stream was created by this endpoint. Usually unset for + // servers. + LastLocalStreamCreatedTimestamp time.Time + // The last time a stream was created by the remote endpoint. Usually unset + // for clients. + LastRemoteStreamCreatedTimestamp time.Time + // The last time a message was sent by this endpoint. + LastMessageSentTimestamp time.Time + // The last time a message was received by this endpoint. + LastMessageReceivedTimestamp time.Time + // The amount of window, granted to the local endpoint by the remote endpoint. + // This may be slightly out of date due to network latency. This does NOT + // include stream level or TCP level flow control info. + LocalFlowControlWindow int64 + // The amount of window, granted to the remote endpoint by the local endpoint. + // This may be slightly out of date due to network latency. This does NOT + // include stream level or TCP level flow control info. + RemoteFlowControlWindow int64 + // The locally bound address. + LocalAddr net.Addr + // The remote bound address. May be absent. + RemoteAddr net.Addr + // Optional, represents the name of the remote endpoint, if different than + // the original target name. + RemoteName string + //TODO: socket options + //TODO: Security +} + +// Socket is the interface that should be satisfied in order to be tracked by +// channelz as Socket. +type Socket interface { + ChannelzMetric() *SocketInternalMetric +} + +type listenSocket struct { + refName string + s Socket + id int64 + pid int64 + cm *channelMap +} + +func (ls *listenSocket) addChild(id int64, e entry) { + grpclog.Errorf("cannot add a child (id = %d) of type %T to a listen socket", id, e) +} + +func (ls *listenSocket) deleteChild(id int64) { + grpclog.Errorf("cannot delete a child (id = %d) from a listen socket", id) +} + +func (ls *listenSocket) triggerDelete() { + ls.cm.deleteEntry(ls.id) + ls.cm.findEntry(ls.pid).deleteChild(ls.id) +} + +func (ls *listenSocket) deleteSelfIfReady() { + grpclog.Errorf("cannot call deleteSelfIfReady on a listen socket") +} + +type normalSocket struct { + refName string + s Socket + id int64 + pid int64 + cm *channelMap +} + +func (ns *normalSocket) addChild(id int64, e entry) { + grpclog.Errorf("cannot add a child (id = %d) of type %T to a normal socket", id, e) +} + +func (ns *normalSocket) deleteChild(id int64) { + grpclog.Errorf("cannot delete a child (id = %d) from a normal socket", id) +} + +func (ns *normalSocket) triggerDelete() { + ns.cm.deleteEntry(ns.id) + ns.cm.findEntry(ns.pid).deleteChild(ns.id) +} + +func (ns *normalSocket) deleteSelfIfReady() { + grpclog.Errorf("cannot call deleteSelfIfReady on a normal socket") +} + +// ServerMetric defines the info channelz provides for a specific Server, which +// includes ServerInternalMetric and channelz-specific data, such as channelz id, +// child list, etc. +type ServerMetric struct { + // ID is the channelz id of this server. + ID int64 + // RefName is the human readable reference string of this server. + RefName string + // ServerData contains server internal metric reported by the server through + // ChannelzMetric(). + ServerData *ServerInternalMetric + // ListenSockets tracks the listener socket type children of this server in the + // format of a map from socket channelz id to corresponding reference string. + ListenSockets map[int64]string +} + +// ServerInternalMetric defines the struct that the implementor of Server interface +// should return from ChannelzMetric(). +type ServerInternalMetric struct { + // The number of incoming calls started on the server. + CallsStarted int64 + // The number of incoming calls that have completed with an OK status. + CallsSucceeded int64 + // The number of incoming calls that have a completed with a non-OK status. + CallsFailed int64 + // The last time a call was started on the server. + LastCallStartedTimestamp time.Time + //TODO: trace +} + +// Server is the interface to be satisfied in order to be tracked by channelz as +// Server. +type Server interface { + ChannelzMetric() *ServerInternalMetric +} + +type server struct { + refName string + s Server + closeCalled bool + sockets map[int64]string + listenSockets map[int64]string + id int64 + cm *channelMap +} + +func (s *server) addChild(id int64, e entry) { + switch v := e.(type) { + case *normalSocket: + s.sockets[id] = v.refName + case *listenSocket: + s.listenSockets[id] = v.refName + default: + grpclog.Errorf("cannot add a child (id = %d) of type %T to a server", id, e) + } +} + +func (s *server) deleteChild(id int64) { + delete(s.sockets, id) + delete(s.listenSockets, id) + s.deleteSelfIfReady() +} + +func (s *server) triggerDelete() { + s.closeCalled = true + s.deleteSelfIfReady() +} + +func (s *server) deleteSelfIfReady() { + if !s.closeCalled || len(s.sockets)+len(s.listenSockets) != 0 { + return + } + s.cm.deleteEntry(s.id) +} diff --git a/vendor/google.golang.org/grpc/clientconn.go b/vendor/google.golang.org/grpc/clientconn.go index f6dab4b7721..261d3e38c37 100644 --- a/vendor/google.golang.org/grpc/clientconn.go +++ b/vendor/google.golang.org/grpc/clientconn.go @@ -1,33 +1,18 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -36,26 +21,60 @@ package grpc import ( "errors" "fmt" + "math" "net" + "reflect" "strings" "sync" "time" "golang.org/x/net/context" "golang.org/x/net/trace" + "google.golang.org/grpc/balancer" + _ "google.golang.org/grpc/balancer/roundrobin" // To register roundrobin. + "google.golang.org/grpc/channelz" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/resolver" + _ "google.golang.org/grpc/resolver/dns" // To register dns resolver. + _ "google.golang.org/grpc/resolver/passthrough" // To register passthrough resolver. + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" "google.golang.org/grpc/transport" ) +const ( + // minimum time to give a connection to complete + minConnectTimeout = 20 * time.Second +) + var ( // ErrClientConnClosing indicates that the operation is illegal because // the ClientConn is closing. - ErrClientConnClosing = errors.New("grpc: the client connection is closing") - // ErrClientConnTimeout indicates that the ClientConn cannot establish the - // underlying connections within the specified timeout. - ErrClientConnTimeout = errors.New("grpc: timed out when dialing") + // + // Deprecated: this error should not be relied upon by users; use the status + // code of Canceled instead. + ErrClientConnClosing = status.Error(codes.Canceled, "grpc: the client connection is closing") + // errConnDrain indicates that the connection starts to be drained and does not accept any new RPCs. + errConnDrain = errors.New("grpc: the connection is drained") + // errConnClosing indicates that the connection is closing. + errConnClosing = errors.New("grpc: the connection is closing") + // errConnUnavailable indicates that the connection is unavailable. + errConnUnavailable = errors.New("grpc: the connection is unavailable") + // errBalancerClosed indicates that the balancer is closed. + errBalancerClosed = errors.New("grpc: balancer is closed") + // We use an accessor so that minConnectTimeout can be + // atomically read and updated while testing. + getMinConnectTimeout = func() time.Duration { + return minConnectTimeout + } +) +// The following errors are returned from Dial and DialContext +var ( // errNoTransportSecurity indicates that there is no transport security // being set for ClientConn. Users should either set one or explicitly // call WithInsecure DialOption to disable security. @@ -69,63 +88,177 @@ var ( errCredentialsConflict = errors.New("grpc: transport credentials are set for an insecure connection (grpc.WithTransportCredentials() and grpc.WithInsecure() are both called)") // errNetworkIO indicates that the connection is down due to some network I/O error. errNetworkIO = errors.New("grpc: failed with network I/O error") - // errConnDrain indicates that the connection starts to be drained and does not accept any new RPCs. - errConnDrain = errors.New("grpc: the connection is drained") - // errConnClosing indicates that the connection is closing. - errConnClosing = errors.New("grpc: the connection is closing") - // errConnUnavailable indicates that the connection is unavailable. - errConnUnavailable = errors.New("grpc: the connection is unavailable") - errNoAddr = errors.New("grpc: there is no address available to dial") - // minimum time to give a connection to complete - minConnectTimeout = 20 * time.Second ) // dialOptions configure a Dial call. dialOptions are set by the DialOption // values passed to Dial. type dialOptions struct { - unaryInt UnaryClientInterceptor - streamInt StreamClientInterceptor - codec Codec - cp Compressor - dc Decompressor - bs backoffStrategy - balancer Balancer - block bool - insecure bool - timeout time.Duration - copts transport.ConnectOptions + unaryInt UnaryClientInterceptor + streamInt StreamClientInterceptor + cp Compressor + dc Decompressor + bs backoffStrategy + block bool + insecure bool + timeout time.Duration + scChan <-chan ServiceConfig + copts transport.ConnectOptions + callOptions []CallOption + // This is used by v1 balancer dial option WithBalancer to support v1 + // balancer, and also by WithBalancerName dial option. + balancerBuilder balancer.Builder + // This is to support grpclb. + resolverBuilder resolver.Builder + waitForHandshake bool + channelzParentID int64 +} + +const ( + defaultClientMaxReceiveMessageSize = 1024 * 1024 * 4 + defaultClientMaxSendMessageSize = math.MaxInt32 +) + +// RegisterChannelz turns on channelz service. +// This is an EXPERIMENTAL API. +func RegisterChannelz() { + channelz.TurnOn() } // DialOption configures how we set up the connection. type DialOption func(*dialOptions) -// WithCodec returns a DialOption which sets a codec for message marshaling and unmarshaling. -func WithCodec(c Codec) DialOption { +// WithWaitForHandshake blocks until the initial settings frame is received from the +// server before assigning RPCs to the connection. +// Experimental API. +func WithWaitForHandshake() DialOption { return func(o *dialOptions) { - o.codec = c + o.waitForHandshake = true } } -// WithCompressor returns a DialOption which sets a CompressorGenerator for generating message -// compressor. +// WithWriteBufferSize lets you set the size of write buffer, this determines how much data can be batched +// before doing a write on the wire. +func WithWriteBufferSize(s int) DialOption { + return func(o *dialOptions) { + o.copts.WriteBufferSize = s + } +} + +// WithReadBufferSize lets you set the size of read buffer, this determines how much data can be read at most +// for each read syscall. +func WithReadBufferSize(s int) DialOption { + return func(o *dialOptions) { + o.copts.ReadBufferSize = s + } +} + +// WithInitialWindowSize returns a DialOption which sets the value for initial window size on a stream. +// The lower bound for window size is 64K and any value smaller than that will be ignored. +func WithInitialWindowSize(s int32) DialOption { + return func(o *dialOptions) { + o.copts.InitialWindowSize = s + } +} + +// WithInitialConnWindowSize returns a DialOption which sets the value for initial window size on a connection. +// The lower bound for window size is 64K and any value smaller than that will be ignored. +func WithInitialConnWindowSize(s int32) DialOption { + return func(o *dialOptions) { + o.copts.InitialConnWindowSize = s + } +} + +// WithMaxMsgSize returns a DialOption which sets the maximum message size the client can receive. +// +// Deprecated: use WithDefaultCallOptions(MaxCallRecvMsgSize(s)) instead. +func WithMaxMsgSize(s int) DialOption { + return WithDefaultCallOptions(MaxCallRecvMsgSize(s)) +} + +// WithDefaultCallOptions returns a DialOption which sets the default CallOptions for calls over the connection. +func WithDefaultCallOptions(cos ...CallOption) DialOption { + return func(o *dialOptions) { + o.callOptions = append(o.callOptions, cos...) + } +} + +// WithCodec returns a DialOption which sets a codec for message marshaling and unmarshaling. +// +// Deprecated: use WithDefaultCallOptions(CallCustomCodec(c)) instead. +func WithCodec(c Codec) DialOption { + return WithDefaultCallOptions(CallCustomCodec(c)) +} + +// WithCompressor returns a DialOption which sets a Compressor to use for +// message compression. It has lower priority than the compressor set by +// the UseCompressor CallOption. +// +// Deprecated: use UseCompressor instead. func WithCompressor(cp Compressor) DialOption { return func(o *dialOptions) { o.cp = cp } } -// WithDecompressor returns a DialOption which sets a DecompressorGenerator for generating -// message decompressor. +// WithDecompressor returns a DialOption which sets a Decompressor to use for +// incoming message decompression. If incoming response messages are encoded +// using the decompressor's Type(), it will be used. Otherwise, the message +// encoding will be used to look up the compressor registered via +// encoding.RegisterCompressor, which will then be used to decompress the +// message. If no compressor is registered for the encoding, an Unimplemented +// status error will be returned. +// +// Deprecated: use encoding.RegisterCompressor instead. func WithDecompressor(dc Decompressor) DialOption { return func(o *dialOptions) { o.dc = dc } } -// WithBalancer returns a DialOption which sets a load balancer. +// WithBalancer returns a DialOption which sets a load balancer with the v1 API. +// Name resolver will be ignored if this DialOption is specified. +// +// Deprecated: use the new balancer APIs in balancer package and WithBalancerName. func WithBalancer(b Balancer) DialOption { return func(o *dialOptions) { - o.balancer = b + o.balancerBuilder = &balancerWrapperBuilder{ + b: b, + } + } +} + +// WithBalancerName sets the balancer that the ClientConn will be initialized +// with. Balancer registered with balancerName will be used. This function +// panics if no balancer was registered by balancerName. +// +// The balancer cannot be overridden by balancer option specified by service +// config. +// +// This is an EXPERIMENTAL API. +func WithBalancerName(balancerName string) DialOption { + builder := balancer.Get(balancerName) + if builder == nil { + panic(fmt.Sprintf("grpc.WithBalancerName: no balancer is registered for name %v", balancerName)) + } + return func(o *dialOptions) { + o.balancerBuilder = builder + } +} + +// withResolverBuilder is only for grpclb. +func withResolverBuilder(b resolver.Builder) DialOption { + return func(o *dialOptions) { + o.resolverBuilder = b + } +} + +// WithServiceConfig returns a DialOption which has a channel to read the service configuration. +// +// Deprecated: service config should be received through name resolver, as specified here. +// https://github.com/grpc/grpc/blob/master/doc/service_config.md +func WithServiceConfig(c <-chan ServiceConfig) DialOption { + return func(o *dialOptions) { + o.scChan = c } } @@ -147,7 +280,7 @@ func WithBackoffConfig(b BackoffConfig) DialOption { return withBackoff(b) } -// withBackoff sets the backoff strategy used for retries after a +// withBackoff sets the backoff strategy used for connectRetryNum after a // failed connection attempt. // // This can be exported if arbitrary backoff strategies are allowed by gRPC. @@ -183,7 +316,7 @@ func WithTransportCredentials(creds credentials.TransportCredentials) DialOption } // WithPerRPCCredentials returns a DialOption which sets -// credentials which will place auth state on each outbound RPC. +// credentials and places auth state on each outbound RPC. func WithPerRPCCredentials(creds credentials.PerRPCCredentials) DialOption { return func(o *dialOptions) { o.copts.PerRPCCredentials = append(o.copts.PerRPCCredentials, creds) @@ -192,27 +325,42 @@ func WithPerRPCCredentials(creds credentials.PerRPCCredentials) DialOption { // WithTimeout returns a DialOption that configures a timeout for dialing a ClientConn // initially. This is valid if and only if WithBlock() is present. +// +// Deprecated: use DialContext and context.WithTimeout instead. func WithTimeout(d time.Duration) DialOption { return func(o *dialOptions) { o.timeout = d } } +func withContextDialer(f func(context.Context, string) (net.Conn, error)) DialOption { + return func(o *dialOptions) { + o.copts.Dialer = f + } +} + // WithDialer returns a DialOption that specifies a function to use for dialing network addresses. // If FailOnNonTempDialError() is set to true, and an error is returned by f, gRPC checks the error's // Temporary() method to decide if it should try to reconnect to the network address. func WithDialer(f func(string, time.Duration) (net.Conn, error)) DialOption { - return func(o *dialOptions) { - o.copts.Dialer = func(ctx context.Context, addr string) (net.Conn, error) { + return withContextDialer( + func(ctx context.Context, addr string) (net.Conn, error) { if deadline, ok := ctx.Deadline(); ok { return f(addr, deadline.Sub(time.Now())) } return f(addr, 0) - } + }) +} + +// WithStatsHandler returns a DialOption that specifies the stats handler +// for all the RPCs and underlying network connections in this ClientConn. +func WithStatsHandler(h stats.Handler) DialOption { + return func(o *dialOptions) { + o.copts.StatsHandler = h } } -// FailOnNonTempDialError returns a DialOption that specified if gRPC fails on non-temporary dial errors. +// FailOnNonTempDialError returns a DialOption that specifies if gRPC fails on non-temporary dial errors. // If f is true, and dialer returns a non-temporary error, gRPC will fail the connection to the network // address and won't try to reconnect. // The default value of FailOnNonTempDialError is false. @@ -230,6 +378,13 @@ func WithUserAgent(s string) DialOption { } } +// WithKeepaliveParams returns a DialOption that specifies keepalive parameters for the client transport. +func WithKeepaliveParams(kp keepalive.ClientParameters) DialOption { + return func(o *dialOptions) { + o.copts.KeepaliveParams = kp + } +} + // WithUnaryInterceptor returns a DialOption that specifies the interceptor for unary RPCs. func WithUnaryInterceptor(f UnaryClientInterceptor) DialOption { return func(o *dialOptions) { @@ -244,22 +399,104 @@ func WithStreamInterceptor(f StreamClientInterceptor) DialOption { } } +// WithAuthority returns a DialOption that specifies the value to be used as +// the :authority pseudo-header. This value only works with WithInsecure and +// has no effect if TransportCredentials are present. +func WithAuthority(a string) DialOption { + return func(o *dialOptions) { + o.copts.Authority = a + } +} + +// WithChannelzParentID returns a DialOption that specifies the channelz ID of current ClientConn's +// parent. This function is used in nested channel creation (e.g. grpclb dial). +func WithChannelzParentID(id int64) DialOption { + return func(o *dialOptions) { + o.channelzParentID = id + } +} + // Dial creates a client connection to the given target. func Dial(target string, opts ...DialOption) (*ClientConn, error) { return DialContext(context.Background(), target, opts...) } -// DialContext creates a client connection to the given target. ctx can be used to -// cancel or expire the pending connecting. Once this function returns, the -// cancellation and expiration of ctx will be noop. Users should call ClientConn.Close -// to terminate all the pending operations after this function returns. -// This is the EXPERIMENTAL API. +// DialContext creates a client connection to the given target. By default, it's +// a non-blocking dial (the function won't wait for connections to be +// established, and connecting happens in the background). To make it a blocking +// dial, use WithBlock() dial option. +// +// In the non-blocking case, the ctx does not act against the connection. It +// only controls the setup steps. +// +// In the blocking case, ctx can be used to cancel or expire the pending +// connection. Once this function returns, the cancellation and expiration of +// ctx will be noop. Users should call ClientConn.Close to terminate all the +// pending operations after this function returns. +// +// The target name syntax is defined in +// https://github.com/grpc/grpc/blob/master/doc/naming.md. +// e.g. to use dns resolver, a "dns:///" prefix should be applied to the target. func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *ClientConn, err error) { cc := &ClientConn{ target: target, - conns: make(map[Address]*addrConn), + csMgr: &connectivityStateManager{}, + conns: make(map[*addrConn]struct{}), + + blockingpicker: newPickerWrapper(), } cc.ctx, cc.cancel = context.WithCancel(context.Background()) + + for _, opt := range opts { + opt(&cc.dopts) + } + + if channelz.IsOn() { + if cc.dopts.channelzParentID != 0 { + cc.channelzID = channelz.RegisterChannel(cc, cc.dopts.channelzParentID, target) + } else { + cc.channelzID = channelz.RegisterChannel(cc, 0, target) + } + } + + if !cc.dopts.insecure { + if cc.dopts.copts.TransportCredentials == nil { + return nil, errNoTransportSecurity + } + } else { + if cc.dopts.copts.TransportCredentials != nil { + return nil, errCredentialsConflict + } + for _, cd := range cc.dopts.copts.PerRPCCredentials { + if cd.RequireTransportSecurity() { + return nil, errTransportCredentialsMissing + } + } + } + + cc.mkp = cc.dopts.copts.KeepaliveParams + + if cc.dopts.copts.Dialer == nil { + cc.dopts.copts.Dialer = newProxyDialer( + func(ctx context.Context, addr string) (net.Conn, error) { + network, addr := parseDialTarget(addr) + return dialContext(ctx, network, addr) + }, + ) + } + + if cc.dopts.copts.UserAgent != "" { + cc.dopts.copts.UserAgent += " " + grpcUA + } else { + cc.dopts.copts.UserAgent = grpcUA + } + + if cc.dopts.timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, cc.dopts.timeout) + defer cancel() + } + defer func() { select { case <-ctx.Done(): @@ -272,119 +509,148 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * } }() - for _, opt := range opts { - opt(&cc.dopts) - } - - // Set defaults. - if cc.dopts.codec == nil { - cc.dopts.codec = protoCodec{} + scSet := false + if cc.dopts.scChan != nil { + // Try to get an initial service config. + select { + case sc, ok := <-cc.dopts.scChan: + if ok { + cc.sc = sc + scSet = true + } + default: + } } if cc.dopts.bs == nil { cc.dopts.bs = DefaultBackoffConfig } + if cc.dopts.resolverBuilder == nil { + // Only try to parse target when resolver builder is not already set. + cc.parsedTarget = parseTarget(cc.target) + grpclog.Infof("parsed scheme: %q", cc.parsedTarget.Scheme) + cc.dopts.resolverBuilder = resolver.Get(cc.parsedTarget.Scheme) + if cc.dopts.resolverBuilder == nil { + // If resolver builder is still nil, the parse target's scheme is + // not registered. Fallback to default resolver and set Endpoint to + // the original unparsed target. + grpclog.Infof("scheme %q not registered, fallback to default scheme", cc.parsedTarget.Scheme) + cc.parsedTarget = resolver.Target{ + Scheme: resolver.GetDefaultScheme(), + Endpoint: target, + } + cc.dopts.resolverBuilder = resolver.Get(cc.parsedTarget.Scheme) + } + } else { + cc.parsedTarget = resolver.Target{Endpoint: target} + } creds := cc.dopts.copts.TransportCredentials if creds != nil && creds.Info().ServerName != "" { cc.authority = creds.Info().ServerName + } else if cc.dopts.insecure && cc.dopts.copts.Authority != "" { + cc.authority = cc.dopts.copts.Authority } else { - colonPos := strings.LastIndex(target, ":") - if colonPos == -1 { - colonPos = len(target) - } - cc.authority = target[:colonPos] + // Use endpoint from "scheme://authority/endpoint" as the default + // authority for ClientConn. + cc.authority = cc.parsedTarget.Endpoint } - var ok bool - waitC := make(chan error, 1) - go func() { - var addrs []Address - if cc.dopts.balancer == nil { - // Connect to target directly if balancer is nil. - addrs = append(addrs, Address{Addr: target}) - } else { - var credsClone credentials.TransportCredentials - if creds != nil { - credsClone = creds.Clone() - } - config := BalancerConfig{ - DialCreds: credsClone, - } - if err := cc.dopts.balancer.Start(target, config); err != nil { - waitC <- err - return - } - ch := cc.dopts.balancer.Notify() - if ch == nil { - // There is no name resolver installed. - addrs = append(addrs, Address{Addr: target}) - } else { - addrs, ok = <-ch - if !ok || len(addrs) == 0 { - waitC <- errNoAddr - return - } - } - } - for _, a := range addrs { - if err := cc.resetAddrConn(a, false, nil); err != nil { - waitC <- err - return + + if cc.dopts.scChan != nil && !scSet { + // Blocking wait for the initial service config. + select { + case sc, ok := <-cc.dopts.scChan: + if ok { + cc.sc = sc } + case <-ctx.Done(): + return nil, ctx.Err() } - close(waitC) - }() - var timeoutCh <-chan time.Time - if cc.dopts.timeout > 0 { - timeoutCh = time.After(cc.dopts.timeout) } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case err := <-waitC: - if err != nil { - return nil, err - } - case <-timeoutCh: - return nil, ErrClientConnTimeout + if cc.dopts.scChan != nil { + go cc.scWatcher() + } + + var credsClone credentials.TransportCredentials + if creds := cc.dopts.copts.TransportCredentials; creds != nil { + credsClone = creds.Clone() + } + cc.balancerBuildOpts = balancer.BuildOptions{ + DialCreds: credsClone, + Dialer: cc.dopts.copts.Dialer, + ChannelzParentID: cc.channelzID, + } + + // Build the resolver. + cc.resolverWrapper, err = newCCResolverWrapper(cc) + if err != nil { + return nil, fmt.Errorf("failed to build resolver: %v", err) } - // If balancer is nil or balancer.Notify() is nil, ok will be false here. - // The lbWatcher goroutine will not be created. - if ok { - go cc.lbWatcher() + // Start the resolver wrapper goroutine after resolverWrapper is created. + // + // If the goroutine is started before resolverWrapper is ready, the + // following may happen: The goroutine sends updates to cc. cc forwards + // those to balancer. Balancer creates new addrConn. addrConn fails to + // connect, and calls resolveNow(). resolveNow() tries to use the non-ready + // resolverWrapper. + cc.resolverWrapper.start() + + // A blocking dial blocks until the clientConn is ready. + if cc.dopts.block { + for { + s := cc.GetState() + if s == connectivity.Ready { + break + } + if !cc.WaitForStateChange(ctx, s) { + // ctx got timeout or canceled. + return nil, ctx.Err() + } + } } + return cc, nil } -// ConnectivityState indicates the state of a client connection. -type ConnectivityState int +// connectivityStateManager keeps the connectivity.State of ClientConn. +// This struct will eventually be exported so the balancers can access it. +type connectivityStateManager struct { + mu sync.Mutex + state connectivity.State + notifyChan chan struct{} +} -const ( - // Idle indicates the ClientConn is idle. - Idle ConnectivityState = iota - // Connecting indicates the ClienConn is connecting. - Connecting - // Ready indicates the ClientConn is ready for work. - Ready - // TransientFailure indicates the ClientConn has seen a failure but expects to recover. - TransientFailure - // Shutdown indicates the ClientConn has started shutting down. - Shutdown -) +// updateState updates the connectivity.State of ClientConn. +// If there's a change it notifies goroutines waiting on state change to +// happen. +func (csm *connectivityStateManager) updateState(state connectivity.State) { + csm.mu.Lock() + defer csm.mu.Unlock() + if csm.state == connectivity.Shutdown { + return + } + if csm.state == state { + return + } + csm.state = state + if csm.notifyChan != nil { + // There are other goroutines waiting on this channel. + close(csm.notifyChan) + csm.notifyChan = nil + } +} + +func (csm *connectivityStateManager) getState() connectivity.State { + csm.mu.Lock() + defer csm.mu.Unlock() + return csm.state +} -func (s ConnectivityState) String() string { - switch s { - case Idle: - return "IDLE" - case Connecting: - return "CONNECTING" - case Ready: - return "READY" - case TransientFailure: - return "TRANSIENT_FAILURE" - case Shutdown: - return "SHUTDOWN" - default: - panic(fmt.Sprintf("unknown connectivity state: %d", s)) +func (csm *connectivityStateManager) getNotifyChan() <-chan struct{} { + csm.mu.Lock() + defer csm.mu.Unlock() + if csm.notifyChan == nil { + csm.notifyChan = make(chan struct{}) } + return csm.notifyChan } // ClientConn represents a client connection to an RPC server. @@ -392,191 +658,399 @@ type ClientConn struct { ctx context.Context cancel context.CancelFunc - target string - authority string - dopts dialOptions + target string + parsedTarget resolver.Target + authority string + dopts dialOptions + csMgr *connectivityStateManager + + balancerBuildOpts balancer.BuildOptions + resolverWrapper *ccResolverWrapper + blockingpicker *pickerWrapper mu sync.RWMutex - conns map[Address]*addrConn + sc ServiceConfig + scRaw string + conns map[*addrConn]struct{} + // Keepalive parameter can be updated if a GoAway is received. + mkp keepalive.ClientParameters + curBalancerName string + preBalancerName string // previous balancer name. + curAddresses []resolver.Address + balancerWrapper *ccBalancerWrapper + + channelzID int64 // channelz unique identification number + czmu sync.RWMutex + callsStarted int64 + callsSucceeded int64 + callsFailed int64 + lastCallStartedTime time.Time } -func (cc *ClientConn) lbWatcher() { - for addrs := range cc.dopts.balancer.Notify() { - var ( - add []Address // Addresses need to setup connections. - del []*addrConn // Connections need to tear down. - ) - cc.mu.Lock() +// WaitForStateChange waits until the connectivity.State of ClientConn changes from sourceState or +// ctx expires. A true value is returned in former case and false in latter. +// This is an EXPERIMENTAL API. +func (cc *ClientConn) WaitForStateChange(ctx context.Context, sourceState connectivity.State) bool { + ch := cc.csMgr.getNotifyChan() + if cc.csMgr.getState() != sourceState { + return true + } + select { + case <-ctx.Done(): + return false + case <-ch: + return true + } +} + +// GetState returns the connectivity.State of ClientConn. +// This is an EXPERIMENTAL API. +func (cc *ClientConn) GetState() connectivity.State { + return cc.csMgr.getState() +} + +func (cc *ClientConn) scWatcher() { + for { + select { + case sc, ok := <-cc.dopts.scChan: + if !ok { + return + } + cc.mu.Lock() + // TODO: load balance policy runtime change is ignored. + // We may revist this decision in the future. + cc.sc = sc + cc.scRaw = "" + cc.mu.Unlock() + case <-cc.ctx.Done(): + return + } + } +} + +func (cc *ClientConn) handleResolvedAddrs(addrs []resolver.Address, err error) { + cc.mu.Lock() + defer cc.mu.Unlock() + if cc.conns == nil { + // cc was closed. + return + } + + if reflect.DeepEqual(cc.curAddresses, addrs) { + return + } + + cc.curAddresses = addrs + + if cc.dopts.balancerBuilder == nil { + // Only look at balancer types and switch balancer if balancer dial + // option is not set. + var isGRPCLB bool for _, a := range addrs { - if _, ok := cc.conns[a]; !ok { - add = append(add, a) + if a.Type == resolver.GRPCLB { + isGRPCLB = true + break } } - for k, c := range cc.conns { - var keep bool - for _, a := range addrs { - if k == a { - keep = true - break - } + var newBalancerName string + if isGRPCLB { + newBalancerName = grpclbName + } else { + // Address list doesn't contain grpclb address. Try to pick a + // non-grpclb balancer. + newBalancerName = cc.curBalancerName + // If current balancer is grpclb, switch to the previous one. + if newBalancerName == grpclbName { + newBalancerName = cc.preBalancerName } - if !keep { - del = append(del, c) - delete(cc.conns, c.addr) + // The following could be true in two cases: + // - the first time handling resolved addresses + // (curBalancerName="") + // - the first time handling non-grpclb addresses + // (curBalancerName="grpclb", preBalancerName="") + if newBalancerName == "" { + newBalancerName = PickFirstBalancerName } } + cc.switchBalancer(newBalancerName) + } else if cc.balancerWrapper == nil { + // Balancer dial option was set, and this is the first time handling + // resolved addresses. Build a balancer with dopts.balancerBuilder. + cc.balancerWrapper = newCCBalancerWrapper(cc, cc.dopts.balancerBuilder, cc.balancerBuildOpts) + } + + cc.balancerWrapper.handleResolvedAddrs(addrs, nil) +} + +// switchBalancer starts the switching from current balancer to the balancer +// with the given name. +// +// It will NOT send the current address list to the new balancer. If needed, +// caller of this function should send address list to the new balancer after +// this function returns. +// +// Caller must hold cc.mu. +func (cc *ClientConn) switchBalancer(name string) { + if cc.conns == nil { + return + } + + if strings.ToLower(cc.curBalancerName) == strings.ToLower(name) { + return + } + + grpclog.Infof("ClientConn switching balancer to %q", name) + if cc.dopts.balancerBuilder != nil { + grpclog.Infoln("ignoring balancer switching: Balancer DialOption used instead") + return + } + // TODO(bar switching) change this to two steps: drain and close. + // Keep track of sc in wrapper. + if cc.balancerWrapper != nil { + cc.balancerWrapper.close() + } + // Clear all stickiness state. + cc.blockingpicker.clearStickinessState() + + builder := balancer.Get(name) + if builder == nil { + grpclog.Infof("failed to get balancer builder for: %v, using pick_first instead", name) + builder = newPickfirstBuilder() + } + cc.preBalancerName = cc.curBalancerName + cc.curBalancerName = builder.Name() + cc.balancerWrapper = newCCBalancerWrapper(cc, builder, cc.balancerBuildOpts) +} + +func (cc *ClientConn) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { + cc.mu.Lock() + if cc.conns == nil { cc.mu.Unlock() - for _, a := range add { - cc.resetAddrConn(a, true, nil) - } - for _, c := range del { - c.tearDown(errConnDrain) - } + return } + // TODO(bar switching) send updates to all balancer wrappers when balancer + // gracefully switching is supported. + cc.balancerWrapper.handleSubConnStateChange(sc, s) + cc.mu.Unlock() } -// resetAddrConn creates an addrConn for addr and adds it to cc.conns. -// If there is an old addrConn for addr, it will be torn down, using tearDownErr as the reason. -// If tearDownErr is nil, errConnDrain will be used instead. -func (cc *ClientConn) resetAddrConn(addr Address, skipWait bool, tearDownErr error) error { +// newAddrConn creates an addrConn for addrs and adds it to cc.conns. +// +// Caller needs to make sure len(addrs) > 0. +func (cc *ClientConn) newAddrConn(addrs []resolver.Address) (*addrConn, error) { ac := &addrConn{ cc: cc, - addr: addr, + addrs: addrs, dopts: cc.dopts, } ac.ctx, ac.cancel = context.WithCancel(cc.ctx) - ac.stateCV = sync.NewCond(&ac.mu) - if EnableTracing { - ac.events = trace.NewEventLog("grpc.ClientConn", ac.addr.Addr) + // Track ac in cc. This needs to be done before any getTransport(...) is called. + cc.mu.Lock() + if cc.conns == nil { + cc.mu.Unlock() + return nil, ErrClientConnClosing } - if !ac.dopts.insecure { - if ac.dopts.copts.TransportCredentials == nil { - return errNoTransportSecurity - } - } else { - if ac.dopts.copts.TransportCredentials != nil { - return errCredentialsConflict - } - for _, cd := range ac.dopts.copts.PerRPCCredentials { - if cd.RequireTransportSecurity() { - return errTransportCredentialsMissing - } - } + if channelz.IsOn() { + ac.channelzID = channelz.RegisterSubChannel(ac, cc.channelzID, "") } - // Track ac in cc. This needs to be done before any getTransport(...) is called. + cc.conns[ac] = struct{}{} + cc.mu.Unlock() + return ac, nil +} + +// removeAddrConn removes the addrConn in the subConn from clientConn. +// It also tears down the ac with the given error. +func (cc *ClientConn) removeAddrConn(ac *addrConn, err error) { cc.mu.Lock() if cc.conns == nil { cc.mu.Unlock() - return ErrClientConnClosing + return } - stale := cc.conns[ac.addr] - cc.conns[ac.addr] = ac + delete(cc.conns, ac) cc.mu.Unlock() - if stale != nil { - // There is an addrConn alive on ac.addr already. This could be due to - // 1) a buggy Balancer notifies duplicated Addresses; - // 2) goaway was received, a new ac will replace the old ac. - // The old ac should be deleted from cc.conns, but the - // underlying transport should drain rather than close. - if tearDownErr == nil { - // tearDownErr is nil if resetAddrConn is called by - // 1) Dial - // 2) lbWatcher - // In both cases, the stale ac should drain, not close. - stale.tearDown(errConnDrain) - } else { - stale.tearDown(tearDownErr) - } + ac.tearDown(err) +} + +// ChannelzMetric returns ChannelInternalMetric of current ClientConn. +// This is an EXPERIMENTAL API. +func (cc *ClientConn) ChannelzMetric() *channelz.ChannelInternalMetric { + state := cc.GetState() + cc.czmu.RLock() + defer cc.czmu.RUnlock() + return &channelz.ChannelInternalMetric{ + State: state, + Target: cc.target, + CallsStarted: cc.callsStarted, + CallsSucceeded: cc.callsSucceeded, + CallsFailed: cc.callsFailed, + LastCallStartedTimestamp: cc.lastCallStartedTime, + } +} + +func (cc *ClientConn) incrCallsStarted() { + cc.czmu.Lock() + cc.callsStarted++ + // TODO(yuxuanli): will make this a time.Time pointer improve performance? + cc.lastCallStartedTime = time.Now() + cc.czmu.Unlock() +} + +func (cc *ClientConn) incrCallsSucceeded() { + cc.czmu.Lock() + cc.callsSucceeded++ + cc.czmu.Unlock() +} + +func (cc *ClientConn) incrCallsFailed() { + cc.czmu.Lock() + cc.callsFailed++ + cc.czmu.Unlock() +} + +// connect starts to creating transport and also starts the transport monitor +// goroutine for this ac. +// It does nothing if the ac is not IDLE. +// TODO(bar) Move this to the addrConn section. +// This was part of resetAddrConn, keep it here to make the diff look clean. +func (ac *addrConn) connect() error { + ac.mu.Lock() + if ac.state == connectivity.Shutdown { + ac.mu.Unlock() + return errConnClosing } - // skipWait may overwrite the decision in ac.dopts.block. - if ac.dopts.block && !skipWait { - if err := ac.resetTransport(false); err != nil { + if ac.state != connectivity.Idle { + ac.mu.Unlock() + return nil + } + ac.state = connectivity.Connecting + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) + ac.mu.Unlock() + + // Start a goroutine connecting to the server asynchronously. + go func() { + if err := ac.resetTransport(); err != nil { + grpclog.Warningf("Failed to dial %s: %v; please retry.", ac.addrs[0].Addr, err) if err != errConnClosing { - // Tear down ac and delete it from cc.conns. - cc.mu.Lock() - delete(cc.conns, ac.addr) - cc.mu.Unlock() + // Keep this ac in cc.conns, to get the reason it's torn down. ac.tearDown(err) } - if e, ok := err.(transport.ConnectionError); ok && !e.Temporary() { - return e.Origin() - } - return err + return } - // Start to monitor the error status of transport. - go ac.transportMonitor() - } else { - // Start a goroutine connecting to the server asynchronously. - go func() { - if err := ac.resetTransport(false); err != nil { - grpclog.Printf("Failed to dial %s: %v; please retry.", ac.addr.Addr, err) - if err != errConnClosing { - // Keep this ac in cc.conns, to get the reason it's torn down. - ac.tearDown(err) - } - return - } - ac.transportMonitor() - }() - } + ac.transportMonitor() + }() return nil } -func (cc *ClientConn) getTransport(ctx context.Context, opts BalancerGetOptions) (transport.ClientTransport, func(), error) { - var ( - ac *addrConn - ok bool - put func() - ) - if cc.dopts.balancer == nil { - // If balancer is nil, there should be only one addrConn available. - cc.mu.RLock() - if cc.conns == nil { - cc.mu.RUnlock() - return nil, nil, toRPCErr(ErrClientConnClosing) - } - for _, ac = range cc.conns { - // Break after the first iteration to get the first addrConn. - ok = true +// tryUpdateAddrs tries to update ac.addrs with the new addresses list. +// +// It checks whether current connected address of ac is in the new addrs list. +// - If true, it updates ac.addrs and returns true. The ac will keep using +// the existing connection. +// - If false, it does nothing and returns false. +func (ac *addrConn) tryUpdateAddrs(addrs []resolver.Address) bool { + ac.mu.Lock() + defer ac.mu.Unlock() + grpclog.Infof("addrConn: tryUpdateAddrs curAddr: %v, addrs: %v", ac.curAddr, addrs) + if ac.state == connectivity.Shutdown { + ac.addrs = addrs + return true + } + + var curAddrFound bool + for _, a := range addrs { + if reflect.DeepEqual(ac.curAddr, a) { + curAddrFound = true break } - cc.mu.RUnlock() - } else { - var ( - addr Address - err error - ) - addr, put, err = cc.dopts.balancer.Get(ctx, opts) - if err != nil { - return nil, nil, toRPCErr(err) - } - cc.mu.RLock() - if cc.conns == nil { - cc.mu.RUnlock() - return nil, nil, toRPCErr(ErrClientConnClosing) - } - ac, ok = cc.conns[addr] - cc.mu.RUnlock() } + grpclog.Infof("addrConn: tryUpdateAddrs curAddrFound: %v", curAddrFound) + if curAddrFound { + ac.addrs = addrs + ac.reconnectIdx = 0 // Start reconnecting from beginning in the new list. + } + + return curAddrFound +} + +// GetMethodConfig gets the method config of the input method. +// If there's an exact match for input method (i.e. /service/method), we return +// the corresponding MethodConfig. +// If there isn't an exact match for the input method, we look for the default config +// under the service (i.e /service/). If there is a default MethodConfig for +// the service, we return it. +// Otherwise, we return an empty MethodConfig. +func (cc *ClientConn) GetMethodConfig(method string) MethodConfig { + // TODO: Avoid the locking here. + cc.mu.RLock() + defer cc.mu.RUnlock() + m, ok := cc.sc.Methods[method] if !ok { - if put != nil { - put() - } - return nil, nil, errConnClosing + i := strings.LastIndex(method, "/") + m = cc.sc.Methods[method[:i+1]] } - t, err := ac.wait(ctx, cc.dopts.balancer != nil, !opts.BlockingWait) + return m +} + +func (cc *ClientConn) getTransport(ctx context.Context, failfast bool) (transport.ClientTransport, func(balancer.DoneInfo), error) { + t, done, err := cc.blockingpicker.pick(ctx, failfast, balancer.PickOptions{}) if err != nil { - if put != nil { - put() + return nil, nil, toRPCErr(err) + } + return t, done, nil +} + +// handleServiceConfig parses the service config string in JSON format to Go native +// struct ServiceConfig, and store both the struct and the JSON string in ClientConn. +func (cc *ClientConn) handleServiceConfig(js string) error { + sc, err := parseServiceConfig(js) + if err != nil { + return err + } + cc.mu.Lock() + cc.scRaw = js + cc.sc = sc + if sc.LB != nil && *sc.LB != grpclbName { // "grpclb" is not a valid balancer option in service config. + if cc.curBalancerName == grpclbName { + // If current balancer is grpclb, there's at least one grpclb + // balancer address in the resolved list. Don't switch the balancer, + // but change the previous balancer name, so if a new resolved + // address list doesn't contain grpclb address, balancer will be + // switched to *sc.LB. + cc.preBalancerName = *sc.LB + } else { + cc.switchBalancer(*sc.LB) + cc.balancerWrapper.handleResolvedAddrs(cc.curAddresses, nil) + } + } + + if envConfigStickinessOn { + var newStickinessMDKey string + if sc.stickinessMetadataKey != nil && *sc.stickinessMetadataKey != "" { + newStickinessMDKey = *sc.stickinessMetadataKey } - return nil, nil, err + // newStickinessMDKey is "" if one of the following happens: + // - stickinessMetadataKey is set to "" + // - stickinessMetadataKey field doesn't exist in service config + cc.blockingpicker.updateStickinessMDKey(strings.ToLower(newStickinessMDKey)) + } + + cc.mu.Unlock() + return nil +} + +func (cc *ClientConn) resolveNow(o resolver.ResolveNowOption) { + cc.mu.Lock() + r := cc.resolverWrapper + cc.mu.Unlock() + if r == nil { + return } - return t, put, nil + go r.resolveNow(o) } // Close tears down the ClientConn and all underlying connections. func (cc *ClientConn) Close() error { - cc.cancel() + defer cc.cancel() cc.mu.Lock() if cc.conns == nil { @@ -585,13 +1059,29 @@ func (cc *ClientConn) Close() error { } conns := cc.conns cc.conns = nil + cc.csMgr.updateState(connectivity.Shutdown) + + rWrapper := cc.resolverWrapper + cc.resolverWrapper = nil + bWrapper := cc.balancerWrapper + cc.balancerWrapper = nil cc.mu.Unlock() - if cc.dopts.balancer != nil { - cc.dopts.balancer.Close() + + cc.blockingpicker.close() + + if rWrapper != nil { + rWrapper.close() + } + if bWrapper != nil { + bWrapper.close() } - for _, ac := range conns { + + for ac := range conns { ac.tearDown(ErrClientConnClosing) } + if channelz.IsOn() { + channelz.RemoveEntry(cc.channelzID) + } return nil } @@ -601,14 +1091,15 @@ type addrConn struct { cancel context.CancelFunc cc *ClientConn - addr Address + addrs []resolver.Address dopts dialOptions events trace.EventLog + acbw balancer.SubConn - mu sync.Mutex - state ConnectivityState - stateCV *sync.Cond - down func(error) // the handler called when a connection is down. + mu sync.Mutex + curAddr resolver.Address + reconnectIdx int // The index in addrs list to start reconnecting from. + state connectivity.State // ready is closed and becomes nil when a new transport is up or failed // due to timeout. ready chan struct{} @@ -616,6 +1107,35 @@ type addrConn struct { // The reason this addrConn is torn down. tearDownErr error + + connectRetryNum int + // backoffDeadline is the time until which resetTransport needs to + // wait before increasing connectRetryNum count. + backoffDeadline time.Time + // connectDeadline is the time by which all connection + // negotiations must complete. + connectDeadline time.Time + + channelzID int64 // channelz unique identification number + czmu sync.RWMutex + callsStarted int64 + callsSucceeded int64 + callsFailed int64 + lastCallStartedTime time.Time +} + +// adjustParams updates parameters used to create transports upon +// receiving a GoAway. +func (ac *addrConn) adjustParams(r transport.GoAwayReason) { + switch r { + case transport.GoAwayTooManyPings: + v := 2 * ac.dopts.copts.KeepaliveParams.Time + ac.cc.mu.Lock() + if v > ac.cc.mkp.Time { + ac.cc.mkp.Time = v + } + ac.cc.mu.Unlock() + } } // printf records an event in ac's event log, unless ac has been closed. @@ -634,198 +1154,282 @@ func (ac *addrConn) errorf(format string, a ...interface{}) { } } -// getState returns the connectivity state of the Conn -func (ac *addrConn) getState() ConnectivityState { - ac.mu.Lock() - defer ac.mu.Unlock() - return ac.state -} - -// waitForStateChange blocks until the state changes to something other than the sourceState. -func (ac *addrConn) waitForStateChange(ctx context.Context, sourceState ConnectivityState) (ConnectivityState, error) { +// resetTransport recreates a transport to the address for ac. The old +// transport will close itself on error or when the clientconn is closed. +// The created transport must receive initial settings frame from the server. +// In case that doesn't happen, transportMonitor will kill the newly created +// transport after connectDeadline has expired. +// In case there was an error on the transport before the settings frame was +// received, resetTransport resumes connecting to backends after the one that +// was previously connected to. In case end of the list is reached, resetTransport +// backs off until the original deadline. +// If the DialOption WithWaitForHandshake was set, resetTrasport returns +// successfully only after server settings are received. +// +// TODO(bar) make sure all state transitions are valid. +func (ac *addrConn) resetTransport() error { ac.mu.Lock() - defer ac.mu.Unlock() - if sourceState != ac.state { - return ac.state, nil + if ac.state == connectivity.Shutdown { + ac.mu.Unlock() + return errConnClosing } - done := make(chan struct{}) - var err error - go func() { - select { - case <-ctx.Done(): - ac.mu.Lock() - err = ctx.Err() - ac.stateCV.Broadcast() - ac.mu.Unlock() - case <-done: - } - }() - defer close(done) - for sourceState == ac.state { - ac.stateCV.Wait() - if err != nil { - return ac.state, err - } + if ac.ready != nil { + close(ac.ready) + ac.ready = nil } - return ac.state, nil -} - -func (ac *addrConn) resetTransport(closeTransport bool) error { - for retries := 0; ; retries++ { + ac.transport = nil + ridx := ac.reconnectIdx + ac.mu.Unlock() + ac.cc.mu.RLock() + ac.dopts.copts.KeepaliveParams = ac.cc.mkp + ac.cc.mu.RUnlock() + var backoffDeadline, connectDeadline time.Time + for connectRetryNum := 0; ; connectRetryNum++ { ac.mu.Lock() - ac.printf("connecting") - if ac.state == Shutdown { - // ac.tearDown(...) has been invoked. + if ac.backoffDeadline.IsZero() { + // This means either a successful HTTP2 connection was established + // or this is the first time this addrConn is trying to establish a + // connection. + backoffFor := ac.dopts.bs.backoff(connectRetryNum) // time.Duration. + // This will be the duration that dial gets to finish. + dialDuration := getMinConnectTimeout() + if backoffFor > dialDuration { + // Give dial more time as we keep failing to connect. + dialDuration = backoffFor + } + start := time.Now() + backoffDeadline = start.Add(backoffFor) + connectDeadline = start.Add(dialDuration) + ridx = 0 // Start connecting from the beginning. + } else { + // Continue trying to connect with the same deadlines. + connectRetryNum = ac.connectRetryNum + backoffDeadline = ac.backoffDeadline + connectDeadline = ac.connectDeadline + ac.backoffDeadline = time.Time{} + ac.connectDeadline = time.Time{} + ac.connectRetryNum = 0 + } + if ac.state == connectivity.Shutdown { ac.mu.Unlock() return errConnClosing } - if ac.down != nil { - ac.down(downErrorf(false, true, "%v", errNetworkIO)) - ac.down = nil + ac.printf("connecting") + if ac.state != connectivity.Connecting { + ac.state = connectivity.Connecting + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) } - ac.state = Connecting - ac.stateCV.Broadcast() - t := ac.transport + // copy ac.addrs in case of race + addrsIter := make([]resolver.Address, len(ac.addrs)) + copy(addrsIter, ac.addrs) + copts := ac.dopts.copts ac.mu.Unlock() - if closeTransport && t != nil { - t.Close() + connected, err := ac.createTransport(connectRetryNum, ridx, backoffDeadline, connectDeadline, addrsIter, copts) + if err != nil { + return err + } + if connected { + return nil } - sleepTime := ac.dopts.bs.backoff(retries) - timeout := minConnectTimeout - if timeout < sleepTime { - timeout = sleepTime + } +} + +// createTransport creates a connection to one of the backends in addrs. +// It returns true if a connection was established. +func (ac *addrConn) createTransport(connectRetryNum, ridx int, backoffDeadline, connectDeadline time.Time, addrs []resolver.Address, copts transport.ConnectOptions) (bool, error) { + for i := ridx; i < len(addrs); i++ { + addr := addrs[i] + target := transport.TargetInfo{ + Addr: addr.Addr, + Metadata: addr.Metadata, + Authority: ac.cc.authority, + } + done := make(chan struct{}) + onPrefaceReceipt := func() { + ac.mu.Lock() + close(done) + if !ac.backoffDeadline.IsZero() { + // If we haven't already started reconnecting to + // other backends. + // Note, this can happen when writer notices an error + // and triggers resetTransport while at the same time + // reader receives the preface and invokes this closure. + ac.backoffDeadline = time.Time{} + ac.connectDeadline = time.Time{} + ac.connectRetryNum = 0 + } + ac.mu.Unlock() } - ctx, cancel := context.WithTimeout(ac.ctx, timeout) - connectTime := time.Now() - sinfo := transport.TargetInfo{ - Addr: ac.addr.Addr, - Metadata: ac.addr.Metadata, + // Do not cancel in the success path because of + // this issue in Go1.6: https://github.com/golang/go/issues/15078. + connectCtx, cancel := context.WithDeadline(ac.ctx, connectDeadline) + if channelz.IsOn() { + copts.ChannelzParentID = ac.channelzID } - newTransport, err := transport.NewClientTransport(ctx, sinfo, ac.dopts.copts) + newTr, err := transport.NewClientTransport(connectCtx, ac.cc.ctx, target, copts, onPrefaceReceipt) if err != nil { cancel() - - if e, ok := err.(transport.ConnectionError); ok && !e.Temporary() { - return err - } - grpclog.Printf("grpc: addrConn.resetTransport failed to create client transport: %v; Reconnecting to %v", err, ac.addr) + ac.cc.blockingpicker.updateConnectionError(err) ac.mu.Lock() - if ac.state == Shutdown { + if ac.state == connectivity.Shutdown { // ac.tearDown(...) has been invoked. ac.mu.Unlock() - return errConnClosing - } - ac.errorf("transient failure: %v", err) - ac.state = TransientFailure - ac.stateCV.Broadcast() - if ac.ready != nil { - close(ac.ready) - ac.ready = nil + return false, errConnClosing } ac.mu.Unlock() - closeTransport = false + grpclog.Warningf("grpc: addrConn.createTransport failed to connect to %v. Err :%v. Reconnecting...", addr, err) + continue + } + if ac.dopts.waitForHandshake { select { - case <-time.After(sleepTime - time.Since(connectTime)): + case <-done: + case <-connectCtx.Done(): + // Didn't receive server preface, must kill this new transport now. + grpclog.Warningf("grpc: addrConn.createTransport failed to receive server preface before deadline.") + newTr.Close() + break case <-ac.ctx.Done(): - return ac.ctx.Err() } - continue } ac.mu.Lock() - ac.printf("ready") - if ac.state == Shutdown { - // ac.tearDown(...) has been invoked. + if ac.state == connectivity.Shutdown { ac.mu.Unlock() - newTransport.Close() - return errConnClosing + // ac.tearDonn(...) has been invoked. + newTr.Close() + return false, errConnClosing } - ac.state = Ready - ac.stateCV.Broadcast() - ac.transport = newTransport + ac.printf("ready") + ac.state = connectivity.Ready + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) + ac.transport = newTr + ac.curAddr = addr if ac.ready != nil { close(ac.ready) ac.ready = nil } - if ac.cc.dopts.balancer != nil { - ac.down = ac.cc.dopts.balancer.Up(ac.addr) + select { + case <-done: + // If the server has responded back with preface already, + // don't set the reconnect parameters. + default: + ac.connectRetryNum = connectRetryNum + ac.backoffDeadline = backoffDeadline + ac.connectDeadline = connectDeadline + ac.reconnectIdx = i + 1 // Start reconnecting from the next backend in the list. } ac.mu.Unlock() - return nil + return true, nil } + ac.mu.Lock() + if ac.state == connectivity.Shutdown { + ac.mu.Unlock() + return false, errConnClosing + } + ac.state = connectivity.TransientFailure + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) + ac.cc.resolveNow(resolver.ResolveNowOption{}) + if ac.ready != nil { + close(ac.ready) + ac.ready = nil + } + ac.mu.Unlock() + timer := time.NewTimer(backoffDeadline.Sub(time.Now())) + select { + case <-timer.C: + case <-ac.ctx.Done(): + timer.Stop() + return false, ac.ctx.Err() + } + return false, nil } // Run in a goroutine to track the error in transport and create the // new transport if an error happens. It returns when the channel is closing. func (ac *addrConn) transportMonitor() { for { + var timer *time.Timer + var cdeadline <-chan time.Time ac.mu.Lock() t := ac.transport + if !ac.connectDeadline.IsZero() { + timer = time.NewTimer(ac.connectDeadline.Sub(time.Now())) + cdeadline = timer.C + } ac.mu.Unlock() + // Block until we receive a goaway or an error occurs. select { - // This is needed to detect the teardown when - // the addrConn is idle (i.e., no RPC in flight). - case <-ac.ctx.Done(): - select { - case <-t.Error(): - t.Close() - default: - } - return case <-t.GoAway(): - // If GoAway happens without any network I/O error, ac is closed without shutting down the - // underlying transport (the transport will be closed when all the pending RPCs finished or - // failed.). - // If GoAway and some network I/O error happen concurrently, ac and its underlying transport - // are closed. - // In both cases, a new ac is created. - select { - case <-t.Error(): - ac.cc.resetAddrConn(ac.addr, true, errNetworkIO) - default: - ac.cc.resetAddrConn(ac.addr, true, errConnDrain) - } - return + done := t.Error() + cleanup := t.Close + // Since this transport will be orphaned (won't have a transportMonitor) + // we need to launch a goroutine to keep track of clientConn.Close() + // happening since it might not be noticed by any other goroutine for a while. + go func() { + <-done + cleanup() + }() case <-t.Error(): - select { - case <-ac.ctx.Done(): - t.Close() - return - case <-t.GoAway(): - ac.cc.resetAddrConn(ac.addr, true, errNetworkIO) - return - default: - } + // In case this is triggered because clientConn.Close() + // was called, we want to immeditately close the transport + // since no other goroutine might notice it for a while. + t.Close() + case <-cdeadline: ac.mu.Lock() - if ac.state == Shutdown { - // ac has been shutdown. + // This implies that client received server preface. + if ac.backoffDeadline.IsZero() { ac.mu.Unlock() - return + continue } - ac.state = TransientFailure - ac.stateCV.Broadcast() ac.mu.Unlock() - if err := ac.resetTransport(true); err != nil { - ac.mu.Lock() - ac.printf("transport exiting: %v", err) - ac.mu.Unlock() - grpclog.Printf("grpc: addrConn.transportMonitor exits due to: %v", err) - if err != errConnClosing { - // Keep this ac in cc.conns, to get the reason it's torn down. - ac.tearDown(err) - } - return + timer = nil + // No server preface received until deadline. + // Kill the connection. + grpclog.Warningf("grpc: addrConn.transportMonitor didn't get server preface after waiting. Closing the new transport now.") + t.Close() + } + if timer != nil { + timer.Stop() + } + // If a GoAway happened, regardless of error, adjust our keepalive + // parameters as appropriate. + select { + case <-t.GoAway(): + ac.adjustParams(t.GetGoAwayReason()) + default: + } + ac.mu.Lock() + if ac.state == connectivity.Shutdown { + ac.mu.Unlock() + return + } + // Set connectivity state to TransientFailure before calling + // resetTransport. Transition READY->CONNECTING is not valid. + ac.state = connectivity.TransientFailure + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) + ac.cc.resolveNow(resolver.ResolveNowOption{}) + ac.curAddr = resolver.Address{} + ac.mu.Unlock() + if err := ac.resetTransport(); err != nil { + ac.mu.Lock() + ac.printf("transport exiting: %v", err) + ac.mu.Unlock() + grpclog.Warningf("grpc: addrConn.transportMonitor exits due to: %v", err) + if err != errConnClosing { + // Keep this ac in cc.conns, to get the reason it's torn down. + ac.tearDown(err) } + return } } } // wait blocks until i) the new transport is up or ii) ctx is done or iii) ac is closed or -// iv) transport is in TransientFailure and there is a balancer/failfast is true. +// iv) transport is in connectivity.TransientFailure and there is a balancer/failfast is true. func (ac *addrConn) wait(ctx context.Context, hasBalancer, failfast bool) (transport.ClientTransport, error) { for { ac.mu.Lock() switch { - case ac.state == Shutdown: + case ac.state == connectivity.Shutdown: if failfast || !hasBalancer { // RPC is failfast or balancer is nil. This RPC should fail with ac.tearDownErr. err := ac.tearDownErr @@ -834,11 +1438,11 @@ func (ac *addrConn) wait(ctx context.Context, hasBalancer, failfast bool) (trans } ac.mu.Unlock() return nil, errConnClosing - case ac.state == Ready: + case ac.state == connectivity.Ready: ct := ac.transport ac.mu.Unlock() return ct, nil - case ac.state == TransientFailure: + case ac.state == connectivity.TransientFailure: if failfast || hasBalancer { ac.mu.Unlock() return nil, errConnUnavailable @@ -859,6 +1463,28 @@ func (ac *addrConn) wait(ctx context.Context, hasBalancer, failfast bool) (trans } } +// getReadyTransport returns the transport if ac's state is READY. +// Otherwise it returns nil, false. +// If ac's state is IDLE, it will trigger ac to connect. +func (ac *addrConn) getReadyTransport() (transport.ClientTransport, bool) { + ac.mu.Lock() + if ac.state == connectivity.Ready { + t := ac.transport + ac.mu.Unlock() + return t, true + } + var idle bool + if ac.state == connectivity.Idle { + idle = true + } + ac.mu.Unlock() + // Trigger idle ac to connect. + if idle { + ac.connect() + } + return nil, false +} + // tearDown starts to tear down the addrConn. // TODO(zhaoq): Make this synchronous to avoid unbounded memory consumption in // some edge cases (e.g., the caller opens and closes many addrConn's in a @@ -866,13 +1492,12 @@ func (ac *addrConn) wait(ctx context.Context, hasBalancer, failfast bool) (trans // tearDown doesn't remove ac from ac.cc.conns. func (ac *addrConn) tearDown(err error) { ac.cancel() - ac.mu.Lock() defer ac.mu.Unlock() - if ac.down != nil { - ac.down(downErrorf(false, false, "%v", err)) - ac.down = nil + if ac.state == connectivity.Shutdown { + return } + ac.curAddr = resolver.Address{} if err == errConnDrain && ac.transport != nil { // GracefulClose(...) may be executed multiple times when // i) receiving multiple GoAway frames from the server; or @@ -880,12 +1505,9 @@ func (ac *addrConn) tearDown(err error) { // address removal and GoAway. ac.transport.GracefulClose() } - if ac.state == Shutdown { - return - } - ac.state = Shutdown + ac.state = connectivity.Shutdown ac.tearDownErr = err - ac.stateCV.Broadcast() + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) if ac.events != nil { ac.events.Finish() ac.events = nil @@ -894,8 +1516,63 @@ func (ac *addrConn) tearDown(err error) { close(ac.ready) ac.ready = nil } - if ac.transport != nil && err != errConnDrain { - ac.transport.Close() + if channelz.IsOn() { + channelz.RemoveEntry(ac.channelzID) } +} + +func (ac *addrConn) getState() connectivity.State { + ac.mu.Lock() + defer ac.mu.Unlock() + return ac.state +} + +func (ac *addrConn) getCurAddr() (ret resolver.Address) { + ac.mu.Lock() + ret = ac.curAddr + ac.mu.Unlock() return } + +func (ac *addrConn) ChannelzMetric() *channelz.ChannelInternalMetric { + ac.mu.Lock() + addr := ac.curAddr.Addr + ac.mu.Unlock() + state := ac.getState() + ac.czmu.RLock() + defer ac.czmu.RUnlock() + return &channelz.ChannelInternalMetric{ + State: state, + Target: addr, + CallsStarted: ac.callsStarted, + CallsSucceeded: ac.callsSucceeded, + CallsFailed: ac.callsFailed, + LastCallStartedTimestamp: ac.lastCallStartedTime, + } +} + +func (ac *addrConn) incrCallsStarted() { + ac.czmu.Lock() + ac.callsStarted++ + ac.lastCallStartedTime = time.Now() + ac.czmu.Unlock() +} + +func (ac *addrConn) incrCallsSucceeded() { + ac.czmu.Lock() + ac.callsSucceeded++ + ac.czmu.Unlock() +} + +func (ac *addrConn) incrCallsFailed() { + ac.czmu.Lock() + ac.callsFailed++ + ac.czmu.Unlock() +} + +// ErrClientConnTimeout indicates that the ClientConn cannot establish the +// underlying connections within the specified timeout. +// +// Deprecated: This error is never returned by grpc and should not be +// referenced by users. +var ErrClientConnTimeout = errors.New("grpc: timed out when dialing") diff --git a/vendor/google.golang.org/grpc/codec.go b/vendor/google.golang.org/grpc/codec.go new file mode 100644 index 00000000000..12977654781 --- /dev/null +++ b/vendor/google.golang.org/grpc/codec.go @@ -0,0 +1,50 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "google.golang.org/grpc/encoding" + _ "google.golang.org/grpc/encoding/proto" // to register the Codec for "proto" +) + +// baseCodec contains the functionality of both Codec and encoding.Codec, but +// omits the name/string, which vary between the two and are not needed for +// anything besides the registry in the encoding package. +type baseCodec interface { + Marshal(v interface{}) ([]byte, error) + Unmarshal(data []byte, v interface{}) error +} + +var _ baseCodec = Codec(nil) +var _ baseCodec = encoding.Codec(nil) + +// Codec defines the interface gRPC uses to encode and decode messages. +// Note that implementations of this interface must be thread safe; +// a Codec's methods can be called from concurrent goroutines. +// +// Deprecated: use encoding.Codec instead. +type Codec interface { + // Marshal returns the wire format of v. + Marshal(v interface{}) ([]byte, error) + // Unmarshal parses the wire format into v. + Unmarshal(data []byte, v interface{}) error + // String returns the name of the Codec implementation. This is unused by + // gRPC. + String() string +} diff --git a/vendor/google.golang.org/grpc/codegen.sh b/vendor/google.golang.org/grpc/codegen.sh index b0094888429..4cdc6ba7c09 100755 --- a/vendor/google.golang.org/grpc/codegen.sh +++ b/vendor/google.golang.org/grpc/codegen.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script serves as an example to demonstrate how to generate the gRPC-Go # interface and the related messages from .proto file. diff --git a/vendor/google.golang.org/grpc/codes/code_string.go b/vendor/google.golang.org/grpc/codes/code_string.go index e6762d08455..0b206a57822 100644 --- a/vendor/google.golang.org/grpc/codes/code_string.go +++ b/vendor/google.golang.org/grpc/codes/code_string.go @@ -1,16 +1,62 @@ -// generated by stringer -type=Code; DO NOT EDIT +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package codes -import "fmt" +import "strconv" -const _Code_name = "OKCanceledUnknownInvalidArgumentDeadlineExceededNotFoundAlreadyExistsPermissionDeniedResourceExhaustedFailedPreconditionAbortedOutOfRangeUnimplementedInternalUnavailableDataLossUnauthenticated" - -var _Code_index = [...]uint8{0, 2, 10, 17, 32, 48, 56, 69, 85, 102, 120, 127, 137, 150, 158, 169, 177, 192} - -func (i Code) String() string { - if i+1 >= Code(len(_Code_index)) { - return fmt.Sprintf("Code(%d)", i) +func (c Code) String() string { + switch c { + case OK: + return "OK" + case Canceled: + return "Canceled" + case Unknown: + return "Unknown" + case InvalidArgument: + return "InvalidArgument" + case DeadlineExceeded: + return "DeadlineExceeded" + case NotFound: + return "NotFound" + case AlreadyExists: + return "AlreadyExists" + case PermissionDenied: + return "PermissionDenied" + case ResourceExhausted: + return "ResourceExhausted" + case FailedPrecondition: + return "FailedPrecondition" + case Aborted: + return "Aborted" + case OutOfRange: + return "OutOfRange" + case Unimplemented: + return "Unimplemented" + case Internal: + return "Internal" + case Unavailable: + return "Unavailable" + case DataLoss: + return "DataLoss" + case Unauthenticated: + return "Unauthenticated" + default: + return "Code(" + strconv.FormatInt(int64(c), 10) + ")" } - return _Code_name[_Code_index[i]:_Code_index[i+1]] } diff --git a/vendor/google.golang.org/grpc/codes/codes.go b/vendor/google.golang.org/grpc/codes/codes.go index e14b464acfd..a8280ae660d 100644 --- a/vendor/google.golang.org/grpc/codes/codes.go +++ b/vendor/google.golang.org/grpc/codes/codes.go @@ -1,33 +1,18 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -35,21 +20,23 @@ // consistent across various languages. package codes // import "google.golang.org/grpc/codes" +import ( + "fmt" +) + // A Code is an unsigned 32-bit error code as defined in the gRPC spec. type Code uint32 -//go:generate stringer -type=Code - const ( // OK is returned on success. OK Code = 0 - // Canceled indicates the operation was cancelled (typically by the caller). + // Canceled indicates the operation was canceled (typically by the caller). Canceled Code = 1 - // Unknown error. An example of where this error may be returned is + // Unknown error. An example of where this error may be returned is // if a Status value received from another address space belongs to - // an error-space that is not known in this address space. Also + // an error-space that is not known in this address space. Also // errors raised by APIs that do not return enough error information // may be converted to this error. Unknown Code = 2 @@ -78,15 +65,11 @@ const ( // PermissionDenied indicates the caller does not have permission to // execute the specified operation. It must not be used for rejections // caused by exhausting some resource (use ResourceExhausted - // instead for those errors). It must not be + // instead for those errors). It must not be // used if the caller cannot be identified (use Unauthenticated // instead for those errors). PermissionDenied Code = 7 - // Unauthenticated indicates the request does not have valid - // authentication credentials for the operation. - Unauthenticated Code = 16 - // ResourceExhausted indicates some resource has been exhausted, perhaps // a per-user quota, or perhaps the entire file system is out of space. ResourceExhausted Code = 8 @@ -102,7 +85,7 @@ const ( // (b) Use Aborted if the client should retry at a higher-level // (e.g., restarting a read-modify-write sequence). // (c) Use FailedPrecondition if the client should not retry until - // the system state has been explicitly fixed. E.g., if an "rmdir" + // the system state has been explicitly fixed. E.g., if an "rmdir" // fails because the directory is non-empty, FailedPrecondition // should be returned since the client should not retry unless // they have first fixed up the directory by deleting files from it. @@ -131,7 +114,7 @@ const ( // file size. // // There is a fair bit of overlap between FailedPrecondition and - // OutOfRange. We recommend using OutOfRange (the more specific + // OutOfRange. We recommend using OutOfRange (the more specific // error) when it applies so that callers who are iterating through // a space can easily look for an OutOfRange error to detect when // they are done. @@ -141,8 +124,8 @@ const ( // supported/enabled in this service. Unimplemented Code = 12 - // Internal errors. Means some invariants expected by underlying - // system has been broken. If you see one of these errors, + // Internal errors. Means some invariants expected by underlying + // system has been broken. If you see one of these errors, // something is very broken. Internal Code = 13 @@ -156,4 +139,46 @@ const ( // DataLoss indicates unrecoverable data loss or corruption. DataLoss Code = 15 + + // Unauthenticated indicates the request does not have valid + // authentication credentials for the operation. + Unauthenticated Code = 16 ) + +var strToCode = map[string]Code{ + `"OK"`: OK, + `"CANCELLED"`:/* [sic] */ Canceled, + `"UNKNOWN"`: Unknown, + `"INVALID_ARGUMENT"`: InvalidArgument, + `"DEADLINE_EXCEEDED"`: DeadlineExceeded, + `"NOT_FOUND"`: NotFound, + `"ALREADY_EXISTS"`: AlreadyExists, + `"PERMISSION_DENIED"`: PermissionDenied, + `"RESOURCE_EXHAUSTED"`: ResourceExhausted, + `"FAILED_PRECONDITION"`: FailedPrecondition, + `"ABORTED"`: Aborted, + `"OUT_OF_RANGE"`: OutOfRange, + `"UNIMPLEMENTED"`: Unimplemented, + `"INTERNAL"`: Internal, + `"UNAVAILABLE"`: Unavailable, + `"DATA_LOSS"`: DataLoss, + `"UNAUTHENTICATED"`: Unauthenticated, +} + +// UnmarshalJSON unmarshals b into the Code. +func (c *Code) UnmarshalJSON(b []byte) error { + // From json.Unmarshaler: By convention, to approximate the behavior of + // Unmarshal itself, Unmarshalers implement UnmarshalJSON([]byte("null")) as + // a no-op. + if string(b) == "null" { + return nil + } + if c == nil { + return fmt.Errorf("nil receiver passed to UnmarshalJSON") + } + if jc, ok := strToCode[string(b)]; ok { + *c = jc + return nil + } + return fmt.Errorf("invalid code: %q", string(b)) +} diff --git a/vendor/google.golang.org/grpc/coverage.sh b/vendor/google.golang.org/grpc/coverage.sh deleted file mode 100755 index 120235374a4..00000000000 --- a/vendor/google.golang.org/grpc/coverage.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -set -e - -workdir=.cover -profile="$workdir/cover.out" -mode=set -end2endtest="google.golang.org/grpc/test" - -generate_cover_data() { - rm -rf "$workdir" - mkdir "$workdir" - - for pkg in "$@"; do - if [ $pkg == "google.golang.org/grpc" -o $pkg == "google.golang.org/grpc/transport" -o $pkg == "google.golang.org/grpc/metadata" -o $pkg == "google.golang.org/grpc/credentials" ] - then - f="$workdir/$(echo $pkg | tr / -)" - go test -covermode="$mode" -coverprofile="$f.cover" "$pkg" - go test -covermode="$mode" -coverpkg "$pkg" -coverprofile="$f.e2e.cover" "$end2endtest" - fi - done - - echo "mode: $mode" >"$profile" - grep -h -v "^mode:" "$workdir"/*.cover >>"$profile" -} - -show_cover_report() { - go tool cover -${1}="$profile" -} - -push_to_coveralls() { - goveralls -coverprofile="$profile" -} - -generate_cover_data $(go list ./...) -show_cover_report func -case "$1" in -"") - ;; ---html) - show_cover_report html ;; ---coveralls) - push_to_coveralls ;; -*) - echo >&2 "error: invalid option: $1" ;; -esac -rm -rf "$workdir" diff --git a/vendor/google.golang.org/grpc/credentials/credentials.go b/vendor/google.golang.org/grpc/credentials/credentials.go index 5555ef024f6..3351bf0ee5f 100644 --- a/vendor/google.golang.org/grpc/credentials/credentials.go +++ b/vendor/google.golang.org/grpc/credentials/credentials.go @@ -1,33 +1,18 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -49,10 +34,8 @@ import ( "golang.org/x/net/context" ) -var ( - // alpnProtoStr are the specified application level protocols for gRPC. - alpnProtoStr = []string{"h2"} -) +// alpnProtoStr are the specified application level protocols for gRPC. +var alpnProtoStr = []string{"h2"} // PerRPCCredentials defines the common interface for the credentials which need to // attach security information to every RPC (e.g., oauth2). @@ -60,8 +43,9 @@ type PerRPCCredentials interface { // GetRequestMetadata gets the current request metadata, refreshing // tokens if required. This should be called by the transport layer on // each request, and the data should be populated in headers or other - // context. uri is the URI of the entry point for the request. When - // supported by the underlying implementation, ctx can be used for + // context. If a status code is returned, it will be used as the status + // for the RPC. uri is the URI of the entry point for the request. + // When supported by the underlying implementation, ctx can be used for // timeout and cancellation. // TODO(zhaoq): Define the set of the qualified keys instead of leaving // it as an arbitrary string. @@ -89,11 +73,9 @@ type AuthInfo interface { AuthType() string } -var ( - // ErrConnDispatched indicates that rawConn has been dispatched out of gRPC - // and the caller should not close rawConn. - ErrConnDispatched = errors.New("credentials: rawConn is dispatched out of gRPC") -) +// ErrConnDispatched indicates that rawConn has been dispatched out of gRPC +// and the caller should not close rawConn. +var ErrConnDispatched = errors.New("credentials: rawConn is dispatched out of gRPC") // TransportCredentials defines the common interface for all the live gRPC wire // protocols and supported transport security protocols (e.g., TLS, SSL). @@ -102,10 +84,18 @@ type TransportCredentials interface { // authentication protocol on rawConn for clients. It returns the authenticated // connection and the corresponding auth information about the connection. // Implementations must use the provided context to implement timely cancellation. + // gRPC will try to reconnect if the error returned is a temporary error + // (io.EOF, context.DeadlineExceeded or err.Temporary() == true). + // If the returned error is a wrapper error, implementations should make sure that + // the error implements Temporary() to have the correct retry behaviors. + // + // If the returned net.Conn is closed, it MUST close the net.Conn provided. ClientHandshake(context.Context, string, net.Conn) (net.Conn, AuthInfo, error) // ServerHandshake does the authentication handshake for servers. It returns // the authenticated connection and the corresponding auth information about // the connection. + // + // If the returned net.Conn is closed, it MUST close the net.Conn provided. ServerHandshake(net.Conn) (net.Conn, AuthInfo, error) // Info provides the ProtocolInfo of this TransportCredentials. Info() ProtocolInfo @@ -142,15 +132,15 @@ func (c tlsCreds) Info() ProtocolInfo { } } -func (c *tlsCreds) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (_ net.Conn, _ AuthInfo, err error) { +func (c *tlsCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (_ net.Conn, _ AuthInfo, err error) { // use local cfg to avoid clobbering ServerName if using multiple endpoints cfg := cloneTLSConfig(c.config) if cfg.ServerName == "" { - colonPos := strings.LastIndex(addr, ":") + colonPos := strings.LastIndex(authority, ":") if colonPos == -1 { - colonPos = len(addr) + colonPos = len(authority) } - cfg.ServerName = addr[:colonPos] + cfg.ServerName = authority[:colonPos] } conn := tls.Client(rawConn, cfg) errChannel := make(chan error, 1) @@ -165,9 +155,7 @@ func (c *tlsCreds) ClientHandshake(ctx context.Context, addr string, rawConn net case <-ctx.Done(): return nil, nil, ctx.Err() } - // TODO(zhaoq): Omit the auth info for client now. It is more for - // information than anything else. - return conn, nil, nil + return conn, TLSInfo{conn.ConnectionState()}, nil } func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) { @@ -194,14 +182,14 @@ func NewTLS(c *tls.Config) TransportCredentials { return tc } -// NewClientTLSFromCert constructs a TLS from the input certificate for client. +// NewClientTLSFromCert constructs TLS credentials from the input certificate for client. // serverNameOverride is for testing only. If set to a non empty string, // it will override the virtual host name of authority (e.g. :authority header field) in requests. func NewClientTLSFromCert(cp *x509.CertPool, serverNameOverride string) TransportCredentials { return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}) } -// NewClientTLSFromFile constructs a TLS from the input certificate file for client. +// NewClientTLSFromFile constructs TLS credentials from the input certificate file for client. // serverNameOverride is for testing only. If set to a non empty string, // it will override the virtual host name of authority (e.g. :authority header field) in requests. func NewClientTLSFromFile(certFile, serverNameOverride string) (TransportCredentials, error) { @@ -216,12 +204,12 @@ func NewClientTLSFromFile(certFile, serverNameOverride string) (TransportCredent return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}), nil } -// NewServerTLSFromCert constructs a TLS from the input certificate for server. +// NewServerTLSFromCert constructs TLS credentials from the input certificate for server. func NewServerTLSFromCert(cert *tls.Certificate) TransportCredentials { return NewTLS(&tls.Config{Certificates: []tls.Certificate{*cert}}) } -// NewServerTLSFromFile constructs a TLS from the input certificate file and key +// NewServerTLSFromFile constructs TLS credentials from the input certificate file and key // file for server. func NewServerTLSFromFile(certFile, keyFile string) (TransportCredentials, error) { cert, err := tls.LoadX509KeyPair(certFile, keyFile) diff --git a/vendor/google.golang.org/grpc/credentials/credentials_util_go17.go b/vendor/google.golang.org/grpc/credentials/credentials_util_go17.go index 9647b9ec83a..60409aac0fb 100644 --- a/vendor/google.golang.org/grpc/credentials/credentials_util_go17.go +++ b/vendor/google.golang.org/grpc/credentials/credentials_util_go17.go @@ -1,35 +1,21 @@ // +build go1.7 +// +build !go1.8 /* * - * Copyright 2016, Google Inc. - * All rights reserved. + * Copyright 2016 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -44,8 +30,6 @@ import ( // contains a mutex and must not be copied. // // If cfg is nil, a new zero tls.Config is returned. -// -// TODO replace this function with official clone function. func cloneTLSConfig(cfg *tls.Config) *tls.Config { if cfg == nil { return &tls.Config{} diff --git a/vendor/google.golang.org/grpc/credentials/credentials_util_go18.go b/vendor/google.golang.org/grpc/credentials/credentials_util_go18.go new file mode 100644 index 00000000000..93f0e1d8de2 --- /dev/null +++ b/vendor/google.golang.org/grpc/credentials/credentials_util_go18.go @@ -0,0 +1,38 @@ +// +build go1.8 + +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "crypto/tls" +) + +// cloneTLSConfig returns a shallow clone of the exported +// fields of cfg, ignoring the unexported sync.Once, which +// contains a mutex and must not be copied. +// +// If cfg is nil, a new zero tls.Config is returned. +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + + return cfg.Clone() +} diff --git a/vendor/google.golang.org/grpc/credentials/credentials_util_pre_go17.go b/vendor/google.golang.org/grpc/credentials/credentials_util_pre_go17.go index 09b8d12c79b..d6bbcc9fdd9 100644 --- a/vendor/google.golang.org/grpc/credentials/credentials_util_pre_go17.go +++ b/vendor/google.golang.org/grpc/credentials/credentials_util_pre_go17.go @@ -2,34 +2,19 @@ /* * - * Copyright 2016, Google Inc. - * All rights reserved. + * Copyright 2016 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -44,8 +29,6 @@ import ( // contains a mutex and must not be copied. // // If cfg is nil, a new zero tls.Config is returned. -// -// TODO replace this function with official clone function. func cloneTLSConfig(cfg *tls.Config) *tls.Config { if cfg == nil { return &tls.Config{} diff --git a/vendor/google.golang.org/grpc/doc.go b/vendor/google.golang.org/grpc/doc.go index a35f218852b..187adbb117f 100644 --- a/vendor/google.golang.org/grpc/doc.go +++ b/vendor/google.golang.org/grpc/doc.go @@ -1,6 +1,24 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + /* Package grpc implements an RPC system called gRPC. -See www.grpc.io for more information about gRPC. +See grpc.io for more information about gRPC. */ package grpc // import "google.golang.org/grpc" diff --git a/vendor/google.golang.org/grpc/encoding/encoding.go b/vendor/google.golang.org/grpc/encoding/encoding.go new file mode 100644 index 00000000000..ade8b7cec73 --- /dev/null +++ b/vendor/google.golang.org/grpc/encoding/encoding.go @@ -0,0 +1,118 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package encoding defines the interface for the compressor and codec, and +// functions to register and retrieve compressors and codecs. +// +// This package is EXPERIMENTAL. +package encoding + +import ( + "io" + "strings" +) + +// Identity specifies the optional encoding for uncompressed streams. +// It is intended for grpc internal use only. +const Identity = "identity" + +// Compressor is used for compressing and decompressing when sending or +// receiving messages. +type Compressor interface { + // Compress writes the data written to wc to w after compressing it. If an + // error occurs while initializing the compressor, that error is returned + // instead. + Compress(w io.Writer) (io.WriteCloser, error) + // Decompress reads data from r, decompresses it, and provides the + // uncompressed data via the returned io.Reader. If an error occurs while + // initializing the decompressor, that error is returned instead. + Decompress(r io.Reader) (io.Reader, error) + // Name is the name of the compression codec and is used to set the content + // coding header. The result must be static; the result cannot change + // between calls. + Name() string +} + +var registeredCompressor = make(map[string]Compressor) + +// RegisterCompressor registers the compressor with gRPC by its name. It can +// be activated when sending an RPC via grpc.UseCompressor(). It will be +// automatically accessed when receiving a message based on the content coding +// header. Servers also use it to send a response with the same encoding as +// the request. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Compressors are +// registered with the same name, the one registered last will take effect. +func RegisterCompressor(c Compressor) { + registeredCompressor[c.Name()] = c +} + +// GetCompressor returns Compressor for the given compressor name. +func GetCompressor(name string) Compressor { + return registeredCompressor[name] +} + +// Codec defines the interface gRPC uses to encode and decode messages. Note +// that implementations of this interface must be thread safe; a Codec's +// methods can be called from concurrent goroutines. +type Codec interface { + // Marshal returns the wire format of v. + Marshal(v interface{}) ([]byte, error) + // Unmarshal parses the wire format into v. + Unmarshal(data []byte, v interface{}) error + // Name returns the name of the Codec implementation. The returned string + // will be used as part of content type in transmission. The result must be + // static; the result cannot change between calls. + Name() string +} + +var registeredCodecs = make(map[string]Codec) + +// RegisterCodec registers the provided Codec for use with all gRPC clients and +// servers. +// +// The Codec will be stored and looked up by result of its Name() method, which +// should match the content-subtype of the encoding handled by the Codec. This +// is case-insensitive, and is stored and looked up as lowercase. If the +// result of calling Name() is an empty string, RegisterCodec will panic. See +// Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Compressors are +// registered with the same name, the one registered last will take effect. +func RegisterCodec(codec Codec) { + if codec == nil { + panic("cannot register a nil Codec") + } + contentSubtype := strings.ToLower(codec.Name()) + if contentSubtype == "" { + panic("cannot register Codec with empty string result for String()") + } + registeredCodecs[contentSubtype] = codec +} + +// GetCodec gets a registered Codec by content-subtype, or nil if no Codec is +// registered for the content-subtype. +// +// The content-subtype is expected to be lowercase. +func GetCodec(contentSubtype string) Codec { + return registeredCodecs[contentSubtype] +} diff --git a/vendor/google.golang.org/grpc/encoding/proto/proto.go b/vendor/google.golang.org/grpc/encoding/proto/proto.go new file mode 100644 index 00000000000..66b97a6f692 --- /dev/null +++ b/vendor/google.golang.org/grpc/encoding/proto/proto.go @@ -0,0 +1,110 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package proto defines the protobuf codec. Importing this package will +// register the codec. +package proto + +import ( + "math" + "sync" + + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/encoding" +) + +// Name is the name registered for the proto compressor. +const Name = "proto" + +func init() { + encoding.RegisterCodec(codec{}) +} + +// codec is a Codec implementation with protobuf. It is the default codec for gRPC. +type codec struct{} + +type cachedProtoBuffer struct { + lastMarshaledSize uint32 + proto.Buffer +} + +func capToMaxInt32(val int) uint32 { + if val > math.MaxInt32 { + return uint32(math.MaxInt32) + } + return uint32(val) +} + +func marshal(v interface{}, cb *cachedProtoBuffer) ([]byte, error) { + protoMsg := v.(proto.Message) + newSlice := make([]byte, 0, cb.lastMarshaledSize) + + cb.SetBuf(newSlice) + cb.Reset() + if err := cb.Marshal(protoMsg); err != nil { + return nil, err + } + out := cb.Bytes() + cb.lastMarshaledSize = capToMaxInt32(len(out)) + return out, nil +} + +func (codec) Marshal(v interface{}) ([]byte, error) { + if pm, ok := v.(proto.Marshaler); ok { + // object can marshal itself, no need for buffer + return pm.Marshal() + } + + cb := protoBufferPool.Get().(*cachedProtoBuffer) + out, err := marshal(v, cb) + + // put back buffer and lose the ref to the slice + cb.SetBuf(nil) + protoBufferPool.Put(cb) + return out, err +} + +func (codec) Unmarshal(data []byte, v interface{}) error { + protoMsg := v.(proto.Message) + protoMsg.Reset() + + if pu, ok := protoMsg.(proto.Unmarshaler); ok { + // object can unmarshal itself, no need for buffer + return pu.Unmarshal(data) + } + + cb := protoBufferPool.Get().(*cachedProtoBuffer) + cb.SetBuf(data) + err := cb.Unmarshal(protoMsg) + cb.SetBuf(nil) + protoBufferPool.Put(cb) + return err +} + +func (codec) Name() string { + return Name +} + +var protoBufferPool = &sync.Pool{ + New: func() interface{} { + return &cachedProtoBuffer{ + Buffer: proto.Buffer{}, + lastMarshaledSize: 16, + } + }, +} diff --git a/vendor/google.golang.org/grpc/envconfig.go b/vendor/google.golang.org/grpc/envconfig.go new file mode 100644 index 00000000000..d50178e5171 --- /dev/null +++ b/vendor/google.golang.org/grpc/envconfig.go @@ -0,0 +1,37 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "os" + "strings" +) + +const ( + envConfigPrefix = "GRPC_GO_" + envConfigStickinessStr = envConfigPrefix + "STICKINESS" +) + +var ( + envConfigStickinessOn bool +) + +func init() { + envConfigStickinessOn = strings.EqualFold(os.Getenv(envConfigStickinessStr), "on") +} diff --git a/vendor/google.golang.org/grpc/go16.go b/vendor/google.golang.org/grpc/go16.go new file mode 100644 index 00000000000..535ee9356f3 --- /dev/null +++ b/vendor/google.golang.org/grpc/go16.go @@ -0,0 +1,70 @@ +// +build go1.6,!go1.7 + +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "fmt" + "io" + "net" + "net/http" + + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/grpc/transport" +) + +// dialContext connects to the address on the named network. +func dialContext(ctx context.Context, network, address string) (net.Conn, error) { + return (&net.Dialer{Cancel: ctx.Done()}).Dial(network, address) +} + +func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { + req.Cancel = ctx.Done() + if err := req.Write(conn); err != nil { + return fmt.Errorf("failed to write the HTTP request: %v", err) + } + return nil +} + +// toRPCErr converts an error into an error from the status package. +func toRPCErr(err error) error { + if err == nil || err == io.EOF { + return err + } + if _, ok := status.FromError(err); ok { + return err + } + switch e := err.(type) { + case transport.StreamError: + return status.Error(e.Code, e.Desc) + case transport.ConnectionError: + return status.Error(codes.Unavailable, e.Desc) + default: + switch err { + case context.DeadlineExceeded: + return status.Error(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return status.Error(codes.Canceled, err.Error()) + } + } + return status.Error(codes.Unknown, err.Error()) +} diff --git a/vendor/google.golang.org/grpc/go17.go b/vendor/google.golang.org/grpc/go17.go new file mode 100644 index 00000000000..ec676a93c39 --- /dev/null +++ b/vendor/google.golang.org/grpc/go17.go @@ -0,0 +1,71 @@ +// +build go1.7 + +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + + netctx "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/grpc/transport" +) + +// dialContext connects to the address on the named network. +func dialContext(ctx context.Context, network, address string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, address) +} + +func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { + req = req.WithContext(ctx) + if err := req.Write(conn); err != nil { + return fmt.Errorf("failed to write the HTTP request: %v", err) + } + return nil +} + +// toRPCErr converts an error into an error from the status package. +func toRPCErr(err error) error { + if err == nil || err == io.EOF { + return err + } + if _, ok := status.FromError(err); ok { + return err + } + switch e := err.(type) { + case transport.StreamError: + return status.Error(e.Code, e.Desc) + case transport.ConnectionError: + return status.Error(codes.Unavailable, e.Desc) + default: + switch err { + case context.DeadlineExceeded, netctx.DeadlineExceeded: + return status.Error(codes.DeadlineExceeded, err.Error()) + case context.Canceled, netctx.Canceled: + return status.Error(codes.Canceled, err.Error()) + } + } + return status.Error(codes.Unknown, err.Error()) +} diff --git a/vendor/google.golang.org/grpc/grpclb.go b/vendor/google.golang.org/grpc/grpclb.go new file mode 100644 index 00000000000..bc2b4452558 --- /dev/null +++ b/vendor/google.golang.org/grpc/grpclb.go @@ -0,0 +1,341 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/net/context" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + lbpb "google.golang.org/grpc/grpclb/grpc_lb_v1/messages" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" +) + +const ( + lbTokeyKey = "lb-token" + defaultFallbackTimeout = 10 * time.Second + grpclbName = "grpclb" +) + +func convertDuration(d *lbpb.Duration) time.Duration { + if d == nil { + return 0 + } + return time.Duration(d.Seconds)*time.Second + time.Duration(d.Nanos)*time.Nanosecond +} + +// Client API for LoadBalancer service. +// Mostly copied from generated pb.go file. +// To avoid circular dependency. +type loadBalancerClient struct { + cc *ClientConn +} + +func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...CallOption) (*balanceLoadClientStream, error) { + desc := &StreamDesc{ + StreamName: "BalanceLoad", + ServerStreams: true, + ClientStreams: true, + } + stream, err := c.cc.NewStream(ctx, desc, "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...) + if err != nil { + return nil, err + } + x := &balanceLoadClientStream{stream} + return x, nil +} + +type balanceLoadClientStream struct { + ClientStream +} + +func (x *balanceLoadClientStream) Send(m *lbpb.LoadBalanceRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *balanceLoadClientStream) Recv() (*lbpb.LoadBalanceResponse, error) { + m := new(lbpb.LoadBalanceResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func init() { + balancer.Register(newLBBuilder()) +} + +// newLBBuilder creates a builder for grpclb. +func newLBBuilder() balancer.Builder { + return NewLBBuilderWithFallbackTimeout(defaultFallbackTimeout) +} + +// NewLBBuilderWithFallbackTimeout creates a grpclb builder with the given +// fallbackTimeout. If no response is received from the remote balancer within +// fallbackTimeout, the backend addresses from the resolved address list will be +// used. +// +// Only call this function when a non-default fallback timeout is needed. +func NewLBBuilderWithFallbackTimeout(fallbackTimeout time.Duration) balancer.Builder { + return &lbBuilder{ + fallbackTimeout: fallbackTimeout, + } +} + +type lbBuilder struct { + fallbackTimeout time.Duration +} + +func (b *lbBuilder) Name() string { + return grpclbName +} + +func (b *lbBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { + // This generates a manual resolver builder with a random scheme. This + // scheme will be used to dial to remote LB, so we can send filtered address + // updates to remote LB ClientConn using this manual resolver. + scheme := "grpclb_internal_" + strconv.FormatInt(time.Now().UnixNano(), 36) + r := &lbManualResolver{scheme: scheme, ccb: cc} + + var target string + targetSplitted := strings.Split(cc.Target(), ":///") + if len(targetSplitted) < 2 { + target = cc.Target() + } else { + target = targetSplitted[1] + } + + lb := &lbBalancer{ + cc: newLBCacheClientConn(cc), + target: target, + opt: opt, + fallbackTimeout: b.fallbackTimeout, + doneCh: make(chan struct{}), + + manualResolver: r, + csEvltr: &connectivityStateEvaluator{}, + subConns: make(map[resolver.Address]balancer.SubConn), + scStates: make(map[balancer.SubConn]connectivity.State), + picker: &errPicker{err: balancer.ErrNoSubConnAvailable}, + clientStats: &rpcStats{}, + } + + return lb +} + +type lbBalancer struct { + cc *lbCacheClientConn + target string + opt balancer.BuildOptions + fallbackTimeout time.Duration + doneCh chan struct{} + + // manualResolver is used in the remote LB ClientConn inside grpclb. When + // resolved address updates are received by grpclb, filtered updates will be + // send to remote LB ClientConn through this resolver. + manualResolver *lbManualResolver + // The ClientConn to talk to the remote balancer. + ccRemoteLB *ClientConn + + // Support client side load reporting. Each picker gets a reference to this, + // and will update its content. + clientStats *rpcStats + + mu sync.Mutex // guards everything following. + // The full server list including drops, used to check if the newly received + // serverList contains anything new. Each generate picker will also have + // reference to this list to do the first layer pick. + fullServerList []*lbpb.Server + // All backends addresses, with metadata set to nil. This list contains all + // backend addresses in the same order and with the same duplicates as in + // serverlist. When generating picker, a SubConn slice with the same order + // but with only READY SCs will be gerenated. + backendAddrs []resolver.Address + // Roundrobin functionalities. + csEvltr *connectivityStateEvaluator + state connectivity.State + subConns map[resolver.Address]balancer.SubConn // Used to new/remove SubConn. + scStates map[balancer.SubConn]connectivity.State // Used to filter READY SubConns. + picker balancer.Picker + // Support fallback to resolved backend addresses if there's no response + // from remote balancer within fallbackTimeout. + fallbackTimerExpired bool + serverListReceived bool + // resolvedBackendAddrs is resolvedAddrs minus remote balancers. It's set + // when resolved address updates are received, and read in the goroutine + // handling fallback. + resolvedBackendAddrs []resolver.Address +} + +// regeneratePicker takes a snapshot of the balancer, and generates a picker from +// it. The picker +// - always returns ErrTransientFailure if the balancer is in TransientFailure, +// - does two layer roundrobin pick otherwise. +// Caller must hold lb.mu. +func (lb *lbBalancer) regeneratePicker() { + if lb.state == connectivity.TransientFailure { + lb.picker = &errPicker{err: balancer.ErrTransientFailure} + return + } + var readySCs []balancer.SubConn + for _, a := range lb.backendAddrs { + if sc, ok := lb.subConns[a]; ok { + if st, ok := lb.scStates[sc]; ok && st == connectivity.Ready { + readySCs = append(readySCs, sc) + } + } + } + + if len(lb.fullServerList) <= 0 { + if len(readySCs) <= 0 { + lb.picker = &errPicker{err: balancer.ErrNoSubConnAvailable} + return + } + lb.picker = &rrPicker{subConns: readySCs} + return + } + lb.picker = &lbPicker{ + serverList: lb.fullServerList, + subConns: readySCs, + stats: lb.clientStats, + } +} + +func (lb *lbBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { + grpclog.Infof("lbBalancer: handle SubConn state change: %p, %v", sc, s) + lb.mu.Lock() + defer lb.mu.Unlock() + + oldS, ok := lb.scStates[sc] + if !ok { + grpclog.Infof("lbBalancer: got state changes for an unknown SubConn: %p, %v", sc, s) + return + } + lb.scStates[sc] = s + switch s { + case connectivity.Idle: + sc.Connect() + case connectivity.Shutdown: + // When an address was removed by resolver, b called RemoveSubConn but + // kept the sc's state in scStates. Remove state for this sc here. + delete(lb.scStates, sc) + } + + oldAggrState := lb.state + lb.state = lb.csEvltr.recordTransition(oldS, s) + + // Regenerate picker when one of the following happens: + // - this sc became ready from not-ready + // - this sc became not-ready from ready + // - the aggregated state of balancer became TransientFailure from non-TransientFailure + // - the aggregated state of balancer became non-TransientFailure from TransientFailure + if (oldS == connectivity.Ready) != (s == connectivity.Ready) || + (lb.state == connectivity.TransientFailure) != (oldAggrState == connectivity.TransientFailure) { + lb.regeneratePicker() + } + + lb.cc.UpdateBalancerState(lb.state, lb.picker) +} + +// fallbackToBackendsAfter blocks for fallbackTimeout and falls back to use +// resolved backends (backends received from resolver, not from remote balancer) +// if no connection to remote balancers was successful. +func (lb *lbBalancer) fallbackToBackendsAfter(fallbackTimeout time.Duration) { + timer := time.NewTimer(fallbackTimeout) + defer timer.Stop() + select { + case <-timer.C: + case <-lb.doneCh: + return + } + lb.mu.Lock() + if lb.serverListReceived { + lb.mu.Unlock() + return + } + lb.fallbackTimerExpired = true + lb.refreshSubConns(lb.resolvedBackendAddrs) + lb.mu.Unlock() +} + +// HandleResolvedAddrs sends the updated remoteLB addresses to remoteLB +// clientConn. The remoteLB clientConn will handle creating/removing remoteLB +// connections. +func (lb *lbBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) { + grpclog.Infof("lbBalancer: handleResolvedResult: %+v", addrs) + if len(addrs) <= 0 { + return + } + + var remoteBalancerAddrs, backendAddrs []resolver.Address + for _, a := range addrs { + if a.Type == resolver.GRPCLB { + remoteBalancerAddrs = append(remoteBalancerAddrs, a) + } else { + backendAddrs = append(backendAddrs, a) + } + } + + if lb.ccRemoteLB == nil { + if len(remoteBalancerAddrs) <= 0 { + grpclog.Errorf("grpclb: no remote balancer address is available, should never happen") + return + } + // First time receiving resolved addresses, create a cc to remote + // balancers. + lb.dialRemoteLB(remoteBalancerAddrs[0].ServerName) + // Start the fallback goroutine. + go lb.fallbackToBackendsAfter(lb.fallbackTimeout) + } + + // cc to remote balancers uses lb.manualResolver. Send the updated remote + // balancer addresses to it through manualResolver. + lb.manualResolver.NewAddress(remoteBalancerAddrs) + + lb.mu.Lock() + lb.resolvedBackendAddrs = backendAddrs + // If serverListReceived is true, connection to remote balancer was + // successful and there's no need to do fallback anymore. + // If fallbackTimerExpired is false, fallback hasn't happened yet. + if !lb.serverListReceived && lb.fallbackTimerExpired { + // This means we received a new list of resolved backends, and we are + // still in fallback mode. Need to update the list of backends we are + // using to the new list of backends. + lb.refreshSubConns(lb.resolvedBackendAddrs) + } + lb.mu.Unlock() +} + +func (lb *lbBalancer) Close() { + select { + case <-lb.doneCh: + return + default: + } + close(lb.doneCh) + if lb.ccRemoteLB != nil { + lb.ccRemoteLB.Close() + } + lb.cc.close() +} diff --git a/vendor/google.golang.org/grpc/grpclb_picker.go b/vendor/google.golang.org/grpc/grpclb_picker.go new file mode 100644 index 00000000000..872c7ccea0e --- /dev/null +++ b/vendor/google.golang.org/grpc/grpclb_picker.go @@ -0,0 +1,159 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "sync" + "sync/atomic" + + "golang.org/x/net/context" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + lbpb "google.golang.org/grpc/grpclb/grpc_lb_v1/messages" + "google.golang.org/grpc/status" +) + +type rpcStats struct { + NumCallsStarted int64 + NumCallsFinished int64 + NumCallsFinishedWithDropForRateLimiting int64 + NumCallsFinishedWithDropForLoadBalancing int64 + NumCallsFinishedWithClientFailedToSend int64 + NumCallsFinishedKnownReceived int64 +} + +// toClientStats converts rpcStats to lbpb.ClientStats, and clears rpcStats. +func (s *rpcStats) toClientStats() *lbpb.ClientStats { + stats := &lbpb.ClientStats{ + NumCallsStarted: atomic.SwapInt64(&s.NumCallsStarted, 0), + NumCallsFinished: atomic.SwapInt64(&s.NumCallsFinished, 0), + NumCallsFinishedWithDropForRateLimiting: atomic.SwapInt64(&s.NumCallsFinishedWithDropForRateLimiting, 0), + NumCallsFinishedWithDropForLoadBalancing: atomic.SwapInt64(&s.NumCallsFinishedWithDropForLoadBalancing, 0), + NumCallsFinishedWithClientFailedToSend: atomic.SwapInt64(&s.NumCallsFinishedWithClientFailedToSend, 0), + NumCallsFinishedKnownReceived: atomic.SwapInt64(&s.NumCallsFinishedKnownReceived, 0), + } + return stats +} + +func (s *rpcStats) dropForRateLimiting() { + atomic.AddInt64(&s.NumCallsStarted, 1) + atomic.AddInt64(&s.NumCallsFinishedWithDropForRateLimiting, 1) + atomic.AddInt64(&s.NumCallsFinished, 1) +} + +func (s *rpcStats) dropForLoadBalancing() { + atomic.AddInt64(&s.NumCallsStarted, 1) + atomic.AddInt64(&s.NumCallsFinishedWithDropForLoadBalancing, 1) + atomic.AddInt64(&s.NumCallsFinished, 1) +} + +func (s *rpcStats) failedToSend() { + atomic.AddInt64(&s.NumCallsStarted, 1) + atomic.AddInt64(&s.NumCallsFinishedWithClientFailedToSend, 1) + atomic.AddInt64(&s.NumCallsFinished, 1) +} + +func (s *rpcStats) knownReceived() { + atomic.AddInt64(&s.NumCallsStarted, 1) + atomic.AddInt64(&s.NumCallsFinishedKnownReceived, 1) + atomic.AddInt64(&s.NumCallsFinished, 1) +} + +type errPicker struct { + // Pick always returns this err. + err error +} + +func (p *errPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { + return nil, nil, p.err +} + +// rrPicker does roundrobin on subConns. It's typically used when there's no +// response from remote balancer, and grpclb falls back to the resolved +// backends. +// +// It guaranteed that len(subConns) > 0. +type rrPicker struct { + mu sync.Mutex + subConns []balancer.SubConn // The subConns that were READY when taking the snapshot. + subConnsNext int +} + +func (p *rrPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { + p.mu.Lock() + defer p.mu.Unlock() + sc := p.subConns[p.subConnsNext] + p.subConnsNext = (p.subConnsNext + 1) % len(p.subConns) + return sc, nil, nil +} + +// lbPicker does two layers of picks: +// +// First layer: roundrobin on all servers in serverList, including drops and backends. +// - If it picks a drop, the RPC will fail as being dropped. +// - If it picks a backend, do a second layer pick to pick the real backend. +// +// Second layer: roundrobin on all READY backends. +// +// It's guaranteed that len(serverList) > 0. +type lbPicker struct { + mu sync.Mutex + serverList []*lbpb.Server + serverListNext int + subConns []balancer.SubConn // The subConns that were READY when taking the snapshot. + subConnsNext int + + stats *rpcStats +} + +func (p *lbPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { + p.mu.Lock() + defer p.mu.Unlock() + + // Layer one roundrobin on serverList. + s := p.serverList[p.serverListNext] + p.serverListNext = (p.serverListNext + 1) % len(p.serverList) + + // If it's a drop, return an error and fail the RPC. + if s.DropForRateLimiting { + p.stats.dropForRateLimiting() + return nil, nil, status.Errorf(codes.Unavailable, "request dropped by grpclb") + } + if s.DropForLoadBalancing { + p.stats.dropForLoadBalancing() + return nil, nil, status.Errorf(codes.Unavailable, "request dropped by grpclb") + } + + // If not a drop but there's no ready subConns. + if len(p.subConns) <= 0 { + return nil, nil, balancer.ErrNoSubConnAvailable + } + + // Return the next ready subConn in the list, also collect rpc stats. + sc := p.subConns[p.subConnsNext] + p.subConnsNext = (p.subConnsNext + 1) % len(p.subConns) + done := func(info balancer.DoneInfo) { + if !info.BytesSent { + p.stats.failedToSend() + } else if info.BytesReceived { + p.stats.knownReceived() + } + } + return sc, done, nil +} diff --git a/vendor/google.golang.org/grpc/grpclb_remote_balancer.go b/vendor/google.golang.org/grpc/grpclb_remote_balancer.go new file mode 100644 index 00000000000..b8dd4f18ce5 --- /dev/null +++ b/vendor/google.golang.org/grpc/grpclb_remote_balancer.go @@ -0,0 +1,266 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "fmt" + "net" + "reflect" + "time" + + "golang.org/x/net/context" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/channelz" + + "google.golang.org/grpc/connectivity" + lbpb "google.golang.org/grpc/grpclb/grpc_lb_v1/messages" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" +) + +// processServerList updates balaner's internal state, create/remove SubConns +// and regenerates picker using the received serverList. +func (lb *lbBalancer) processServerList(l *lbpb.ServerList) { + grpclog.Infof("lbBalancer: processing server list: %+v", l) + lb.mu.Lock() + defer lb.mu.Unlock() + + // Set serverListReceived to true so fallback will not take effect if it has + // not hit timeout. + lb.serverListReceived = true + + // If the new server list == old server list, do nothing. + if reflect.DeepEqual(lb.fullServerList, l.Servers) { + grpclog.Infof("lbBalancer: new serverlist same as the previous one, ignoring") + return + } + lb.fullServerList = l.Servers + + var backendAddrs []resolver.Address + for _, s := range l.Servers { + if s.DropForLoadBalancing || s.DropForRateLimiting { + continue + } + + md := metadata.Pairs(lbTokeyKey, s.LoadBalanceToken) + ip := net.IP(s.IpAddress) + ipStr := ip.String() + if ip.To4() == nil { + // Add square brackets to ipv6 addresses, otherwise net.Dial() and + // net.SplitHostPort() will return too many colons error. + ipStr = fmt.Sprintf("[%s]", ipStr) + } + addr := resolver.Address{ + Addr: fmt.Sprintf("%s:%d", ipStr, s.Port), + Metadata: &md, + } + + backendAddrs = append(backendAddrs, addr) + } + + // Call refreshSubConns to create/remove SubConns. + lb.refreshSubConns(backendAddrs) + // Regenerate and update picker no matter if there's update on backends (if + // any SubConn will be newed/removed). Because since the full serverList was + // different, there might be updates in drops or pick weights(different + // number of duplicates). We need to update picker with the fulllist. + // + // Now with cache, even if SubConn was newed/removed, there might be no + // state changes. + lb.regeneratePicker() + lb.cc.UpdateBalancerState(lb.state, lb.picker) +} + +// refreshSubConns creates/removes SubConns with backendAddrs. It returns a bool +// indicating whether the backendAddrs are different from the cached +// backendAddrs (whether any SubConn was newed/removed). +// Caller must hold lb.mu. +func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address) bool { + lb.backendAddrs = nil + var backendsUpdated bool + // addrsSet is the set converted from backendAddrs, it's used to quick + // lookup for an address. + addrsSet := make(map[resolver.Address]struct{}) + // Create new SubConns. + for _, addr := range backendAddrs { + addrWithoutMD := addr + addrWithoutMD.Metadata = nil + addrsSet[addrWithoutMD] = struct{}{} + lb.backendAddrs = append(lb.backendAddrs, addrWithoutMD) + + if _, ok := lb.subConns[addrWithoutMD]; !ok { + backendsUpdated = true + + // Use addrWithMD to create the SubConn. + sc, err := lb.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{}) + if err != nil { + grpclog.Warningf("roundrobinBalancer: failed to create new SubConn: %v", err) + continue + } + lb.subConns[addrWithoutMD] = sc // Use the addr without MD as key for the map. + if _, ok := lb.scStates[sc]; !ok { + // Only set state of new sc to IDLE. The state could already be + // READY for cached SubConns. + lb.scStates[sc] = connectivity.Idle + } + sc.Connect() + } + } + + for a, sc := range lb.subConns { + // a was removed by resolver. + if _, ok := addrsSet[a]; !ok { + backendsUpdated = true + + lb.cc.RemoveSubConn(sc) + delete(lb.subConns, a) + // Keep the state of this sc in b.scStates until sc's state becomes Shutdown. + // The entry will be deleted in HandleSubConnStateChange. + } + } + + return backendsUpdated +} + +func (lb *lbBalancer) readServerList(s *balanceLoadClientStream) error { + for { + reply, err := s.Recv() + if err != nil { + return fmt.Errorf("grpclb: failed to recv server list: %v", err) + } + if serverList := reply.GetServerList(); serverList != nil { + lb.processServerList(serverList) + } + } +} + +func (lb *lbBalancer) sendLoadReport(s *balanceLoadClientStream, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + case <-s.Context().Done(): + return + } + stats := lb.clientStats.toClientStats() + t := time.Now() + stats.Timestamp = &lbpb.Timestamp{ + Seconds: t.Unix(), + Nanos: int32(t.Nanosecond()), + } + if err := s.Send(&lbpb.LoadBalanceRequest{ + LoadBalanceRequestType: &lbpb.LoadBalanceRequest_ClientStats{ + ClientStats: stats, + }, + }); err != nil { + return + } + } +} + +func (lb *lbBalancer) callRemoteBalancer() error { + lbClient := &loadBalancerClient{cc: lb.ccRemoteLB} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stream, err := lbClient.BalanceLoad(ctx, FailFast(false)) + if err != nil { + return fmt.Errorf("grpclb: failed to perform RPC to the remote balancer %v", err) + } + + // grpclb handshake on the stream. + initReq := &lbpb.LoadBalanceRequest{ + LoadBalanceRequestType: &lbpb.LoadBalanceRequest_InitialRequest{ + InitialRequest: &lbpb.InitialLoadBalanceRequest{ + Name: lb.target, + }, + }, + } + if err := stream.Send(initReq); err != nil { + return fmt.Errorf("grpclb: failed to send init request: %v", err) + } + reply, err := stream.Recv() + if err != nil { + return fmt.Errorf("grpclb: failed to recv init response: %v", err) + } + initResp := reply.GetInitialResponse() + if initResp == nil { + return fmt.Errorf("grpclb: reply from remote balancer did not include initial response") + } + if initResp.LoadBalancerDelegate != "" { + return fmt.Errorf("grpclb: Delegation is not supported") + } + + go func() { + if d := convertDuration(initResp.ClientStatsReportInterval); d > 0 { + lb.sendLoadReport(stream, d) + } + }() + return lb.readServerList(stream) +} + +func (lb *lbBalancer) watchRemoteBalancer() { + for { + err := lb.callRemoteBalancer() + select { + case <-lb.doneCh: + return + default: + if err != nil { + grpclog.Error(err) + } + } + + } +} + +func (lb *lbBalancer) dialRemoteLB(remoteLBName string) { + var dopts []DialOption + if creds := lb.opt.DialCreds; creds != nil { + if err := creds.OverrideServerName(remoteLBName); err == nil { + dopts = append(dopts, WithTransportCredentials(creds)) + } else { + grpclog.Warningf("grpclb: failed to override the server name in the credentials: %v, using Insecure", err) + dopts = append(dopts, WithInsecure()) + } + } else { + dopts = append(dopts, WithInsecure()) + } + if lb.opt.Dialer != nil { + // WithDialer takes a different type of function, so we instead use a + // special DialOption here. + dopts = append(dopts, withContextDialer(lb.opt.Dialer)) + } + // Explicitly set pickfirst as the balancer. + dopts = append(dopts, WithBalancerName(PickFirstBalancerName)) + dopts = append(dopts, withResolverBuilder(lb.manualResolver)) + if channelz.IsOn() { + dopts = append(dopts, WithChannelzParentID(lb.opt.ChannelzParentID)) + } + + // DialContext using manualResolver.Scheme, which is a random scheme generated + // when init grpclb. The target name is not important. + cc, err := DialContext(context.Background(), "grpclb:///grpclb.server", dopts...) + if err != nil { + grpclog.Fatalf("failed to dial: %v", err) + } + lb.ccRemoteLB = cc + go lb.watchRemoteBalancer() +} diff --git a/vendor/google.golang.org/grpc/grpclb_util.go b/vendor/google.golang.org/grpc/grpclb_util.go new file mode 100644 index 00000000000..063ba9d8590 --- /dev/null +++ b/vendor/google.golang.org/grpc/grpclb_util.go @@ -0,0 +1,214 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "fmt" + "sync" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" +) + +// The parent ClientConn should re-resolve when grpclb loses connection to the +// remote balancer. When the ClientConn inside grpclb gets a TransientFailure, +// it calls lbManualResolver.ResolveNow(), which calls parent ClientConn's +// ResolveNow, and eventually results in re-resolve happening in parent +// ClientConn's resolver (DNS for example). +// +// parent +// ClientConn +// +-----------------------------------------------------------------+ +// | parent +---------------------------------+ | +// | DNS ClientConn | grpclb | | +// | resolver balancerWrapper | | | +// | + + | grpclb grpclb | | +// | | | | ManualResolver ClientConn | | +// | | | | + + | | +// | | | | | | Transient | | +// | | | | | | Failure | | +// | | | | | <--------- | | | +// | | | <--------------- | ResolveNow | | | +// | | <--------- | ResolveNow | | | | | +// | | ResolveNow | | | | | | +// | | | | | | | | +// | + + | + + | | +// | +---------------------------------+ | +// +-----------------------------------------------------------------+ + +// lbManualResolver is used by the ClientConn inside grpclb. It's a manual +// resolver with a special ResolveNow() function. +// +// When ResolveNow() is called, it calls ResolveNow() on the parent ClientConn, +// so when grpclb client lose contact with remote balancers, the parent +// ClientConn's resolver will re-resolve. +type lbManualResolver struct { + scheme string + ccr resolver.ClientConn + + ccb balancer.ClientConn +} + +func (r *lbManualResolver) Build(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOption) (resolver.Resolver, error) { + r.ccr = cc + return r, nil +} + +func (r *lbManualResolver) Scheme() string { + return r.scheme +} + +// ResolveNow calls resolveNow on the parent ClientConn. +func (r *lbManualResolver) ResolveNow(o resolver.ResolveNowOption) { + r.ccb.ResolveNow(o) +} + +// Close is a noop for Resolver. +func (*lbManualResolver) Close() {} + +// NewAddress calls cc.NewAddress. +func (r *lbManualResolver) NewAddress(addrs []resolver.Address) { + r.ccr.NewAddress(addrs) +} + +// NewServiceConfig calls cc.NewServiceConfig. +func (r *lbManualResolver) NewServiceConfig(sc string) { + r.ccr.NewServiceConfig(sc) +} + +const subConnCacheTime = time.Second * 10 + +// lbCacheClientConn is a wrapper balancer.ClientConn with a SubConn cache. +// SubConns will be kept in cache for subConnCacheTime before being removed. +// +// Its new and remove methods are updated to do cache first. +type lbCacheClientConn struct { + cc balancer.ClientConn + timeout time.Duration + + mu sync.Mutex + // subConnCache only keeps subConns that are being deleted. + subConnCache map[resolver.Address]*subConnCacheEntry + subConnToAddr map[balancer.SubConn]resolver.Address +} + +type subConnCacheEntry struct { + sc balancer.SubConn + + cancel func() + abortDeleting bool +} + +func newLBCacheClientConn(cc balancer.ClientConn) *lbCacheClientConn { + return &lbCacheClientConn{ + cc: cc, + timeout: subConnCacheTime, + subConnCache: make(map[resolver.Address]*subConnCacheEntry), + subConnToAddr: make(map[balancer.SubConn]resolver.Address), + } +} + +func (ccc *lbCacheClientConn) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + if len(addrs) != 1 { + return nil, fmt.Errorf("grpclb calling NewSubConn with addrs of length %v", len(addrs)) + } + addrWithoutMD := addrs[0] + addrWithoutMD.Metadata = nil + + ccc.mu.Lock() + defer ccc.mu.Unlock() + if entry, ok := ccc.subConnCache[addrWithoutMD]; ok { + // If entry is in subConnCache, the SubConn was being deleted. + // cancel function will never be nil. + entry.cancel() + delete(ccc.subConnCache, addrWithoutMD) + return entry.sc, nil + } + + scNew, err := ccc.cc.NewSubConn(addrs, opts) + if err != nil { + return nil, err + } + + ccc.subConnToAddr[scNew] = addrWithoutMD + return scNew, nil +} + +func (ccc *lbCacheClientConn) RemoveSubConn(sc balancer.SubConn) { + ccc.mu.Lock() + defer ccc.mu.Unlock() + addr, ok := ccc.subConnToAddr[sc] + if !ok { + return + } + + if entry, ok := ccc.subConnCache[addr]; ok { + if entry.sc != sc { + // This could happen if NewSubConn was called multiple times for the + // same address, and those SubConns are all removed. We remove sc + // immediately here. + delete(ccc.subConnToAddr, sc) + ccc.cc.RemoveSubConn(sc) + } + return + } + + entry := &subConnCacheEntry{ + sc: sc, + } + ccc.subConnCache[addr] = entry + + timer := time.AfterFunc(ccc.timeout, func() { + ccc.mu.Lock() + if entry.abortDeleting { + return + } + ccc.cc.RemoveSubConn(sc) + delete(ccc.subConnToAddr, sc) + delete(ccc.subConnCache, addr) + ccc.mu.Unlock() + }) + entry.cancel = func() { + if !timer.Stop() { + // If stop was not successful, the timer has fired (this can only + // happen in a race). But the deleting function is blocked on ccc.mu + // because the mutex was held by the caller of this function. + // + // Set abortDeleting to true to abort the deleting function. When + // the lock is released, the deleting function will acquire the + // lock, check the value of abortDeleting and return. + entry.abortDeleting = true + } + } +} + +func (ccc *lbCacheClientConn) UpdateBalancerState(s connectivity.State, p balancer.Picker) { + ccc.cc.UpdateBalancerState(s, p) +} + +func (ccc *lbCacheClientConn) close() { + ccc.mu.Lock() + // Only cancel all existing timers. There's no need to remove SubConns. + for _, entry := range ccc.subConnCache { + entry.cancel() + } + ccc.mu.Unlock() +} diff --git a/vendor/google.golang.org/grpc/grpclog/grpclog.go b/vendor/google.golang.org/grpc/grpclog/grpclog.go new file mode 100644 index 00000000000..1fabb11e1ba --- /dev/null +++ b/vendor/google.golang.org/grpc/grpclog/grpclog.go @@ -0,0 +1,126 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpclog defines logging for grpc. +// +// All logs in transport package only go to verbose level 2. +// All logs in other packages in grpc are logged in spite of the verbosity level. +// +// In the default logger, +// severity level can be set by environment variable GRPC_GO_LOG_SEVERITY_LEVEL, +// verbosity level can be set by GRPC_GO_LOG_VERBOSITY_LEVEL. +package grpclog // import "google.golang.org/grpc/grpclog" + +import "os" + +var logger = newLoggerV2() + +// V reports whether verbosity level l is at least the requested verbose level. +func V(l int) bool { + return logger.V(l) +} + +// Info logs to the INFO log. +func Info(args ...interface{}) { + logger.Info(args...) +} + +// Infof logs to the INFO log. Arguments are handled in the manner of fmt.Printf. +func Infof(format string, args ...interface{}) { + logger.Infof(format, args...) +} + +// Infoln logs to the INFO log. Arguments are handled in the manner of fmt.Println. +func Infoln(args ...interface{}) { + logger.Infoln(args...) +} + +// Warning logs to the WARNING log. +func Warning(args ...interface{}) { + logger.Warning(args...) +} + +// Warningf logs to the WARNING log. Arguments are handled in the manner of fmt.Printf. +func Warningf(format string, args ...interface{}) { + logger.Warningf(format, args...) +} + +// Warningln logs to the WARNING log. Arguments are handled in the manner of fmt.Println. +func Warningln(args ...interface{}) { + logger.Warningln(args...) +} + +// Error logs to the ERROR log. +func Error(args ...interface{}) { + logger.Error(args...) +} + +// Errorf logs to the ERROR log. Arguments are handled in the manner of fmt.Printf. +func Errorf(format string, args ...interface{}) { + logger.Errorf(format, args...) +} + +// Errorln logs to the ERROR log. Arguments are handled in the manner of fmt.Println. +func Errorln(args ...interface{}) { + logger.Errorln(args...) +} + +// Fatal logs to the FATAL log. Arguments are handled in the manner of fmt.Print. +// It calls os.Exit() with exit code 1. +func Fatal(args ...interface{}) { + logger.Fatal(args...) + // Make sure fatal logs will exit. + os.Exit(1) +} + +// Fatalf logs to the FATAL log. Arguments are handled in the manner of fmt.Printf. +// It calles os.Exit() with exit code 1. +func Fatalf(format string, args ...interface{}) { + logger.Fatalf(format, args...) + // Make sure fatal logs will exit. + os.Exit(1) +} + +// Fatalln logs to the FATAL log. Arguments are handled in the manner of fmt.Println. +// It calle os.Exit()) with exit code 1. +func Fatalln(args ...interface{}) { + logger.Fatalln(args...) + // Make sure fatal logs will exit. + os.Exit(1) +} + +// Print prints to the logger. Arguments are handled in the manner of fmt.Print. +// +// Deprecated: use Info. +func Print(args ...interface{}) { + logger.Info(args...) +} + +// Printf prints to the logger. Arguments are handled in the manner of fmt.Printf. +// +// Deprecated: use Infof. +func Printf(format string, args ...interface{}) { + logger.Infof(format, args...) +} + +// Println prints to the logger. Arguments are handled in the manner of fmt.Println. +// +// Deprecated: use Infoln. +func Println(args ...interface{}) { + logger.Infoln(args...) +} diff --git a/vendor/google.golang.org/grpc/grpclog/logger.go b/vendor/google.golang.org/grpc/grpclog/logger.go index 3b2933079e9..097494f710f 100644 --- a/vendor/google.golang.org/grpc/grpclog/logger.go +++ b/vendor/google.golang.org/grpc/grpclog/logger.go @@ -1,52 +1,26 @@ /* * - * Copyright 2015, Google Inc. - * All rights reserved. + * Copyright 2015 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ -/* -Package grpclog defines logging for grpc. -*/ -package grpclog // import "google.golang.org/grpc/grpclog" - -import ( - "log" - "os" -) - -// Use golang's standard logger by default. -// Access is not mutex-protected: do not modify except in init() -// functions. -var logger Logger = log.New(os.Stderr, "", log.LstdFlags) +package grpclog // Logger mimics golang's standard Logger as an interface. +// +// Deprecated: use LoggerV2. type Logger interface { Fatal(args ...interface{}) Fatalf(format string, args ...interface{}) @@ -58,36 +32,54 @@ type Logger interface { // SetLogger sets the logger that is used in grpc. Call only from // init() functions. +// +// Deprecated: use SetLoggerV2. func SetLogger(l Logger) { - logger = l + logger = &loggerWrapper{Logger: l} +} + +// loggerWrapper wraps Logger into a LoggerV2. +type loggerWrapper struct { + Logger +} + +func (g *loggerWrapper) Info(args ...interface{}) { + g.Logger.Print(args...) +} + +func (g *loggerWrapper) Infoln(args ...interface{}) { + g.Logger.Println(args...) +} + +func (g *loggerWrapper) Infof(format string, args ...interface{}) { + g.Logger.Printf(format, args...) +} + +func (g *loggerWrapper) Warning(args ...interface{}) { + g.Logger.Print(args...) } -// Fatal is equivalent to Print() followed by a call to os.Exit() with a non-zero exit code. -func Fatal(args ...interface{}) { - logger.Fatal(args...) +func (g *loggerWrapper) Warningln(args ...interface{}) { + g.Logger.Println(args...) } -// Fatalf is equivalent to Printf() followed by a call to os.Exit() with a non-zero exit code. -func Fatalf(format string, args ...interface{}) { - logger.Fatalf(format, args...) +func (g *loggerWrapper) Warningf(format string, args ...interface{}) { + g.Logger.Printf(format, args...) } -// Fatalln is equivalent to Println() followed by a call to os.Exit()) with a non-zero exit code. -func Fatalln(args ...interface{}) { - logger.Fatalln(args...) +func (g *loggerWrapper) Error(args ...interface{}) { + g.Logger.Print(args...) } -// Print prints to the logger. Arguments are handled in the manner of fmt.Print. -func Print(args ...interface{}) { - logger.Print(args...) +func (g *loggerWrapper) Errorln(args ...interface{}) { + g.Logger.Println(args...) } -// Printf prints to the logger. Arguments are handled in the manner of fmt.Printf. -func Printf(format string, args ...interface{}) { - logger.Printf(format, args...) +func (g *loggerWrapper) Errorf(format string, args ...interface{}) { + g.Logger.Printf(format, args...) } -// Println prints to the logger. Arguments are handled in the manner of fmt.Println. -func Println(args ...interface{}) { - logger.Println(args...) +func (g *loggerWrapper) V(l int) bool { + // Returns true for all verbose level. + return true } diff --git a/vendor/google.golang.org/grpc/grpclog/loggerv2.go b/vendor/google.golang.org/grpc/grpclog/loggerv2.go new file mode 100644 index 00000000000..d4932577695 --- /dev/null +++ b/vendor/google.golang.org/grpc/grpclog/loggerv2.go @@ -0,0 +1,195 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclog + +import ( + "io" + "io/ioutil" + "log" + "os" + "strconv" +) + +// LoggerV2 does underlying logging work for grpclog. +type LoggerV2 interface { + // Info logs to INFO log. Arguments are handled in the manner of fmt.Print. + Info(args ...interface{}) + // Infoln logs to INFO log. Arguments are handled in the manner of fmt.Println. + Infoln(args ...interface{}) + // Infof logs to INFO log. Arguments are handled in the manner of fmt.Printf. + Infof(format string, args ...interface{}) + // Warning logs to WARNING log. Arguments are handled in the manner of fmt.Print. + Warning(args ...interface{}) + // Warningln logs to WARNING log. Arguments are handled in the manner of fmt.Println. + Warningln(args ...interface{}) + // Warningf logs to WARNING log. Arguments are handled in the manner of fmt.Printf. + Warningf(format string, args ...interface{}) + // Error logs to ERROR log. Arguments are handled in the manner of fmt.Print. + Error(args ...interface{}) + // Errorln logs to ERROR log. Arguments are handled in the manner of fmt.Println. + Errorln(args ...interface{}) + // Errorf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. + Errorf(format string, args ...interface{}) + // Fatal logs to ERROR log. Arguments are handled in the manner of fmt.Print. + // gRPC ensures that all Fatal logs will exit with os.Exit(1). + // Implementations may also call os.Exit() with a non-zero exit code. + Fatal(args ...interface{}) + // Fatalln logs to ERROR log. Arguments are handled in the manner of fmt.Println. + // gRPC ensures that all Fatal logs will exit with os.Exit(1). + // Implementations may also call os.Exit() with a non-zero exit code. + Fatalln(args ...interface{}) + // Fatalf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. + // gRPC ensures that all Fatal logs will exit with os.Exit(1). + // Implementations may also call os.Exit() with a non-zero exit code. + Fatalf(format string, args ...interface{}) + // V reports whether verbosity level l is at least the requested verbose level. + V(l int) bool +} + +// SetLoggerV2 sets logger that is used in grpc to a V2 logger. +// Not mutex-protected, should be called before any gRPC functions. +func SetLoggerV2(l LoggerV2) { + logger = l +} + +const ( + // infoLog indicates Info severity. + infoLog int = iota + // warningLog indicates Warning severity. + warningLog + // errorLog indicates Error severity. + errorLog + // fatalLog indicates Fatal severity. + fatalLog +) + +// severityName contains the string representation of each severity. +var severityName = []string{ + infoLog: "INFO", + warningLog: "WARNING", + errorLog: "ERROR", + fatalLog: "FATAL", +} + +// loggerT is the default logger used by grpclog. +type loggerT struct { + m []*log.Logger + v int +} + +// NewLoggerV2 creates a loggerV2 with the provided writers. +// Fatal logs will be written to errorW, warningW, infoW, followed by exit(1). +// Error logs will be written to errorW, warningW and infoW. +// Warning logs will be written to warningW and infoW. +// Info logs will be written to infoW. +func NewLoggerV2(infoW, warningW, errorW io.Writer) LoggerV2 { + return NewLoggerV2WithVerbosity(infoW, warningW, errorW, 0) +} + +// NewLoggerV2WithVerbosity creates a loggerV2 with the provided writers and +// verbosity level. +func NewLoggerV2WithVerbosity(infoW, warningW, errorW io.Writer, v int) LoggerV2 { + var m []*log.Logger + m = append(m, log.New(infoW, severityName[infoLog]+": ", log.LstdFlags)) + m = append(m, log.New(io.MultiWriter(infoW, warningW), severityName[warningLog]+": ", log.LstdFlags)) + ew := io.MultiWriter(infoW, warningW, errorW) // ew will be used for error and fatal. + m = append(m, log.New(ew, severityName[errorLog]+": ", log.LstdFlags)) + m = append(m, log.New(ew, severityName[fatalLog]+": ", log.LstdFlags)) + return &loggerT{m: m, v: v} +} + +// newLoggerV2 creates a loggerV2 to be used as default logger. +// All logs are written to stderr. +func newLoggerV2() LoggerV2 { + errorW := ioutil.Discard + warningW := ioutil.Discard + infoW := ioutil.Discard + + logLevel := os.Getenv("GRPC_GO_LOG_SEVERITY_LEVEL") + switch logLevel { + case "", "ERROR", "error": // If env is unset, set level to ERROR. + errorW = os.Stderr + case "WARNING", "warning": + warningW = os.Stderr + case "INFO", "info": + infoW = os.Stderr + } + + var v int + vLevel := os.Getenv("GRPC_GO_LOG_VERBOSITY_LEVEL") + if vl, err := strconv.Atoi(vLevel); err == nil { + v = vl + } + return NewLoggerV2WithVerbosity(infoW, warningW, errorW, v) +} + +func (g *loggerT) Info(args ...interface{}) { + g.m[infoLog].Print(args...) +} + +func (g *loggerT) Infoln(args ...interface{}) { + g.m[infoLog].Println(args...) +} + +func (g *loggerT) Infof(format string, args ...interface{}) { + g.m[infoLog].Printf(format, args...) +} + +func (g *loggerT) Warning(args ...interface{}) { + g.m[warningLog].Print(args...) +} + +func (g *loggerT) Warningln(args ...interface{}) { + g.m[warningLog].Println(args...) +} + +func (g *loggerT) Warningf(format string, args ...interface{}) { + g.m[warningLog].Printf(format, args...) +} + +func (g *loggerT) Error(args ...interface{}) { + g.m[errorLog].Print(args...) +} + +func (g *loggerT) Errorln(args ...interface{}) { + g.m[errorLog].Println(args...) +} + +func (g *loggerT) Errorf(format string, args ...interface{}) { + g.m[errorLog].Printf(format, args...) +} + +func (g *loggerT) Fatal(args ...interface{}) { + g.m[fatalLog].Fatal(args...) + // No need to call os.Exit() again because log.Logger.Fatal() calls os.Exit(). +} + +func (g *loggerT) Fatalln(args ...interface{}) { + g.m[fatalLog].Fatalln(args...) + // No need to call os.Exit() again because log.Logger.Fatal() calls os.Exit(). +} + +func (g *loggerT) Fatalf(format string, args ...interface{}) { + g.m[fatalLog].Fatalf(format, args...) + // No need to call os.Exit() again because log.Logger.Fatal() calls os.Exit(). +} + +func (g *loggerT) V(l int) bool { + return l <= g.v +} diff --git a/vendor/google.golang.org/grpc/interceptor.go b/vendor/google.golang.org/grpc/interceptor.go index 8d932efed7e..1f6ef678035 100644 --- a/vendor/google.golang.org/grpc/interceptor.go +++ b/vendor/google.golang.org/grpc/interceptor.go @@ -1,33 +1,18 @@ /* * - * Copyright 2016, Google Inc. - * All rights reserved. + * Copyright 2016 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -40,17 +25,17 @@ import ( // UnaryInvoker is called by UnaryClientInterceptor to complete RPCs. type UnaryInvoker func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error -// UnaryClientInterceptor intercepts the execution of a unary RPC on the client. inovker is the handler to complete the RPC +// UnaryClientInterceptor intercepts the execution of a unary RPC on the client. invoker is the handler to complete the RPC // and it is the responsibility of the interceptor to call it. -// This is the EXPERIMENTAL API. +// This is an EXPERIMENTAL API. type UnaryClientInterceptor func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error // Streamer is called by StreamClientInterceptor to create a ClientStream. type Streamer func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) // StreamClientInterceptor intercepts the creation of ClientStream. It may return a custom ClientStream to intercept all I/O -// operations. streamer is the handlder to create a ClientStream and it is the responsibility of the interceptor to call it. -// This is the EXPERIMENTAL API. +// operations. streamer is the handler to create a ClientStream and it is the responsibility of the interceptor to call it. +// This is an EXPERIMENTAL API. type StreamClientInterceptor func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) // UnaryServerInfo consists of various information about a unary RPC on @@ -63,7 +48,9 @@ type UnaryServerInfo struct { } // UnaryHandler defines the handler invoked by UnaryServerInterceptor to complete the normal -// execution of a unary RPC. +// execution of a unary RPC. If a UnaryHandler returns an error, it should be produced by the +// status package, or else gRPC will use codes.Unknown as the status code and err.Error() as +// the status message of the RPC. type UnaryHandler func(ctx context.Context, req interface{}) (interface{}, error) // UnaryServerInterceptor provides a hook to intercept the execution of a unary RPC on the server. info diff --git a/vendor/google.golang.org/grpc/metadata/metadata.go b/vendor/google.golang.org/grpc/metadata/metadata.go index 65dc5af57dc..bd2eaf40837 100644 --- a/vendor/google.golang.org/grpc/metadata/metadata.go +++ b/vendor/google.golang.org/grpc/metadata/metadata.go @@ -1,91 +1,59 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ // Package metadata define the structure of the metadata supported by gRPC library. +// Please refer to https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md +// for more information about custom-metadata. package metadata // import "google.golang.org/grpc/metadata" import ( - "encoding/base64" "fmt" "strings" "golang.org/x/net/context" ) -const ( - binHdrSuffix = "-bin" -) - -// encodeKeyValue encodes key and value qualified for transmission via gRPC. -// Transmitting binary headers violates HTTP/2 spec. -// TODO(zhaoq): Maybe check if k is ASCII also. -func encodeKeyValue(k, v string) (string, string) { - k = strings.ToLower(k) - if strings.HasSuffix(k, binHdrSuffix) { - val := base64.StdEncoding.EncodeToString([]byte(v)) - v = string(val) - } - return k, v -} - -// DecodeKeyValue returns the original key and value corresponding to the -// encoded data in k, v. -// If k is a binary header and v contains comma, v is split on comma before decoded, -// and the decoded v will be joined with comma before returned. +// DecodeKeyValue returns k, v, nil. +// +// Deprecated: use k and v directly instead. func DecodeKeyValue(k, v string) (string, string, error) { - if !strings.HasSuffix(k, binHdrSuffix) { - return k, v, nil - } - vvs := strings.Split(v, ",") - for i, vv := range vvs { - val, err := base64.StdEncoding.DecodeString(vv) - if err != nil { - return "", "", err - } - vvs[i] = string(val) - } - return k, strings.Join(vvs, ","), nil + return k, v, nil } // MD is a mapping from metadata keys to values. Users should use the following // two convenience functions New and Pairs to generate MD. type MD map[string][]string -// New creates a MD from given key-value map. +// New creates an MD from a given key-value map. +// +// Only the following ASCII characters are allowed in keys: +// - digits: 0-9 +// - uppercase letters: A-Z (normalized to lower) +// - lowercase letters: a-z +// - special characters: -_. +// Uppercase letters are automatically converted to lowercase. +// +// Keys beginning with "grpc-" are reserved for grpc-internal use only and may +// result in errors if set in metadata. func New(m map[string]string) MD { md := MD{} - for k, v := range m { - key, val := encodeKeyValue(k, v) + for k, val := range m { + key := strings.ToLower(k) md[key] = append(md[key], val) } return md @@ -93,19 +61,28 @@ func New(m map[string]string) MD { // Pairs returns an MD formed by the mapping of key, value ... // Pairs panics if len(kv) is odd. +// +// Only the following ASCII characters are allowed in keys: +// - digits: 0-9 +// - uppercase letters: A-Z (normalized to lower) +// - lowercase letters: a-z +// - special characters: -_. +// Uppercase letters are automatically converted to lowercase. +// +// Keys beginning with "grpc-" are reserved for grpc-internal use only and may +// result in errors if set in metadata. func Pairs(kv ...string) MD { if len(kv)%2 == 1 { panic(fmt.Sprintf("metadata: Pairs got the odd number of input pairs for metadata: %d", len(kv))) } md := MD{} - var k string + var key string for i, s := range kv { if i%2 == 0 { - k = s + key = strings.ToLower(s) continue } - key, val := encodeKeyValue(k, s) - md[key] = append(md[key], val) + md[key] = append(md[key], s) } return md } @@ -120,9 +97,33 @@ func (md MD) Copy() MD { return Join(md) } -// Join joins any number of MDs into a single MD. +// Get obtains the values for a given key. +func (md MD) Get(k string) []string { + k = strings.ToLower(k) + return md[k] +} + +// Set sets the value of a given key with a slice of values. +func (md MD) Set(k string, vals ...string) { + if len(vals) == 0 { + return + } + k = strings.ToLower(k) + md[k] = vals +} + +// Append adds the values to key k, not overwriting what was already stored at that key. +func (md MD) Append(k string, vals ...string) { + if len(vals) == 0 { + return + } + k = strings.ToLower(k) + md[k] = append(md[k], vals...) +} + +// Join joins any number of mds into a single MD. // The order of values for each key is determined by the order in which -// the MDs containing those values are presented to Join. +// the mds containing those values are presented to Join. func Join(mds ...MD) MD { out := MD{} for _, md := range mds { @@ -133,17 +134,77 @@ func Join(mds ...MD) MD { return out } -type mdKey struct{} +type mdIncomingKey struct{} +type mdOutgoingKey struct{} + +// NewIncomingContext creates a new context with incoming md attached. +func NewIncomingContext(ctx context.Context, md MD) context.Context { + return context.WithValue(ctx, mdIncomingKey{}, md) +} + +// NewOutgoingContext creates a new context with outgoing md attached. If used +// in conjunction with AppendToOutgoingContext, NewOutgoingContext will +// overwrite any previously-appended metadata. +func NewOutgoingContext(ctx context.Context, md MD) context.Context { + return context.WithValue(ctx, mdOutgoingKey{}, rawMD{md: md}) +} -// NewContext creates a new context with md attached. -func NewContext(ctx context.Context, md MD) context.Context { - return context.WithValue(ctx, mdKey{}, md) +// AppendToOutgoingContext returns a new context with the provided kv merged +// with any existing metadata in the context. Please refer to the +// documentation of Pairs for a description of kv. +func AppendToOutgoingContext(ctx context.Context, kv ...string) context.Context { + if len(kv)%2 == 1 { + panic(fmt.Sprintf("metadata: AppendToOutgoingContext got an odd number of input pairs for metadata: %d", len(kv))) + } + md, _ := ctx.Value(mdOutgoingKey{}).(rawMD) + added := make([][]string, len(md.added)+1) + copy(added, md.added) + added[len(added)-1] = make([]string, len(kv)) + copy(added[len(added)-1], kv) + return context.WithValue(ctx, mdOutgoingKey{}, rawMD{md: md.md, added: added}) } -// FromContext returns the MD in ctx if it exists. -// The returned md should be immutable, writing to it may cause races. -// Modification should be made to the copies of the returned md. -func FromContext(ctx context.Context) (md MD, ok bool) { - md, ok = ctx.Value(mdKey{}).(MD) +// FromIncomingContext returns the incoming metadata in ctx if it exists. The +// returned MD should not be modified. Writing to it may cause races. +// Modification should be made to copies of the returned MD. +func FromIncomingContext(ctx context.Context) (md MD, ok bool) { + md, ok = ctx.Value(mdIncomingKey{}).(MD) return } + +// FromOutgoingContextRaw returns the un-merged, intermediary contents +// of rawMD. Remember to perform strings.ToLower on the keys. The returned +// MD should not be modified. Writing to it may cause races. Modification +// should be made to copies of the returned MD. +// +// This is intended for gRPC-internal use ONLY. +func FromOutgoingContextRaw(ctx context.Context) (MD, [][]string, bool) { + raw, ok := ctx.Value(mdOutgoingKey{}).(rawMD) + if !ok { + return nil, nil, false + } + + return raw.md, raw.added, true +} + +// FromOutgoingContext returns the outgoing metadata in ctx if it exists. The +// returned MD should not be modified. Writing to it may cause races. +// Modification should be made to copies of the returned MD. +func FromOutgoingContext(ctx context.Context) (MD, bool) { + raw, ok := ctx.Value(mdOutgoingKey{}).(rawMD) + if !ok { + return nil, false + } + + mds := make([]MD, 0, len(raw.added)+1) + mds = append(mds, raw.md) + for _, vv := range raw.added { + mds = append(mds, Pairs(vv...)) + } + return Join(mds...), ok +} + +type rawMD struct { + md MD + added [][]string +} diff --git a/vendor/google.golang.org/grpc/peer/peer.go b/vendor/google.golang.org/grpc/peer/peer.go index bfa6205ba9e..317b8b9d09a 100644 --- a/vendor/google.golang.org/grpc/peer/peer.go +++ b/vendor/google.golang.org/grpc/peer/peer.go @@ -1,33 +1,18 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -42,7 +27,8 @@ import ( "google.golang.org/grpc/credentials" ) -// Peer contains the information of the peer for an RPC. +// Peer contains the information of the peer for an RPC, such as the address +// and authentication information. type Peer struct { // Addr is the peer address. Addr net.Addr diff --git a/vendor/google.golang.org/grpc/picker_wrapper.go b/vendor/google.golang.org/grpc/picker_wrapper.go new file mode 100644 index 00000000000..0a984e6c8af --- /dev/null +++ b/vendor/google.golang.org/grpc/picker_wrapper.go @@ -0,0 +1,331 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "io" + "sync" + "sync/atomic" + + "golang.org/x/net/context" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/channelz" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + "google.golang.org/grpc/transport" +) + +// pickerWrapper is a wrapper of balancer.Picker. It blocks on certain pick +// actions and unblock when there's a picker update. +type pickerWrapper struct { + mu sync.Mutex + done bool + blockingCh chan struct{} + picker balancer.Picker + + // The latest connection happened. + connErrMu sync.Mutex + connErr error + + stickinessMDKey atomic.Value + stickiness *stickyStore +} + +func newPickerWrapper() *pickerWrapper { + bp := &pickerWrapper{ + blockingCh: make(chan struct{}), + stickiness: newStickyStore(), + } + return bp +} + +func (bp *pickerWrapper) updateConnectionError(err error) { + bp.connErrMu.Lock() + bp.connErr = err + bp.connErrMu.Unlock() +} + +func (bp *pickerWrapper) connectionError() error { + bp.connErrMu.Lock() + err := bp.connErr + bp.connErrMu.Unlock() + return err +} + +func (bp *pickerWrapper) updateStickinessMDKey(newKey string) { + // No need to check ok because mdKey == "" if ok == false. + if oldKey, _ := bp.stickinessMDKey.Load().(string); oldKey != newKey { + bp.stickinessMDKey.Store(newKey) + bp.stickiness.reset(newKey) + } +} + +func (bp *pickerWrapper) getStickinessMDKey() string { + // No need to check ok because mdKey == "" if ok == false. + mdKey, _ := bp.stickinessMDKey.Load().(string) + return mdKey +} + +func (bp *pickerWrapper) clearStickinessState() { + if oldKey := bp.getStickinessMDKey(); oldKey != "" { + // There's no need to reset store if mdKey was "". + bp.stickiness.reset(oldKey) + } +} + +// updatePicker is called by UpdateBalancerState. It unblocks all blocked pick. +func (bp *pickerWrapper) updatePicker(p balancer.Picker) { + bp.mu.Lock() + if bp.done { + bp.mu.Unlock() + return + } + bp.picker = p + // bp.blockingCh should never be nil. + close(bp.blockingCh) + bp.blockingCh = make(chan struct{}) + bp.mu.Unlock() +} + +func doneChannelzWrapper(acw *acBalancerWrapper, done func(balancer.DoneInfo)) func(balancer.DoneInfo) { + acw.mu.Lock() + ac := acw.ac + acw.mu.Unlock() + ac.incrCallsStarted() + return func(b balancer.DoneInfo) { + if b.Err != nil && b.Err != io.EOF { + ac.incrCallsFailed() + } else { + ac.incrCallsSucceeded() + } + if done != nil { + done(b) + } + } +} + +// pick returns the transport that will be used for the RPC. +// It may block in the following cases: +// - there's no picker +// - the current picker returns ErrNoSubConnAvailable +// - the current picker returns other errors and failfast is false. +// - the subConn returned by the current picker is not READY +// When one of these situations happens, pick blocks until the picker gets updated. +func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer.PickOptions) (transport.ClientTransport, func(balancer.DoneInfo), error) { + + mdKey := bp.getStickinessMDKey() + stickyKey, isSticky := stickyKeyFromContext(ctx, mdKey) + + // Potential race here: if stickinessMDKey is updated after the above two + // lines, and this pick is a sticky pick, the following put could add an + // entry to sticky store with an outdated sticky key. + // + // The solution: keep the current md key in sticky store, and at the + // beginning of each get/put, check the mdkey against store.curMDKey. + // - Cons: one more string comparing for each get/put. + // - Pros: the string matching happens inside get/put, so the overhead for + // non-sticky RPCs will be minimal. + + if isSticky { + if t, ok := bp.stickiness.get(mdKey, stickyKey); ok { + // Done function returned is always nil. + return t, nil, nil + } + } + + var ( + p balancer.Picker + ch chan struct{} + ) + + for { + bp.mu.Lock() + if bp.done { + bp.mu.Unlock() + return nil, nil, ErrClientConnClosing + } + + if bp.picker == nil { + ch = bp.blockingCh + } + if ch == bp.blockingCh { + // This could happen when either: + // - bp.picker is nil (the previous if condition), or + // - has called pick on the current picker. + bp.mu.Unlock() + select { + case <-ctx.Done(): + return nil, nil, ctx.Err() + case <-ch: + } + continue + } + + ch = bp.blockingCh + p = bp.picker + bp.mu.Unlock() + + subConn, done, err := p.Pick(ctx, opts) + + if err != nil { + switch err { + case balancer.ErrNoSubConnAvailable: + continue + case balancer.ErrTransientFailure: + if !failfast { + continue + } + return nil, nil, status.Errorf(codes.Unavailable, "%v, latest connection error: %v", err, bp.connectionError()) + default: + // err is some other error. + return nil, nil, toRPCErr(err) + } + } + + acw, ok := subConn.(*acBalancerWrapper) + if !ok { + grpclog.Infof("subconn returned from pick is not *acBalancerWrapper") + continue + } + if t, ok := acw.getAddrConn().getReadyTransport(); ok { + if isSticky { + bp.stickiness.put(mdKey, stickyKey, acw) + } + if channelz.IsOn() { + return t, doneChannelzWrapper(acw, done), nil + } + return t, done, nil + } + grpclog.Infof("blockingPicker: the picked transport is not ready, loop back to repick") + // If ok == false, ac.state is not READY. + // A valid picker always returns READY subConn. This means the state of ac + // just changed, and picker will be updated shortly. + // continue back to the beginning of the for loop to repick. + } +} + +func (bp *pickerWrapper) close() { + bp.mu.Lock() + defer bp.mu.Unlock() + if bp.done { + return + } + bp.done = true + close(bp.blockingCh) +} + +type stickyStoreEntry struct { + acw *acBalancerWrapper + addr resolver.Address +} + +type stickyStore struct { + mu sync.Mutex + // curMDKey is check before every get/put to avoid races. The operation will + // abort immediately when the given mdKey is different from the curMDKey. + curMDKey string + store map[string]*stickyStoreEntry +} + +func newStickyStore() *stickyStore { + return &stickyStore{ + store: make(map[string]*stickyStoreEntry), + } +} + +// reset clears the map in stickyStore, and set the currentMDKey to newMDKey. +func (ss *stickyStore) reset(newMDKey string) { + ss.mu.Lock() + ss.curMDKey = newMDKey + ss.store = make(map[string]*stickyStoreEntry) + ss.mu.Unlock() +} + +// stickyKey is the key to look up in store. mdKey will be checked against +// curMDKey to avoid races. +func (ss *stickyStore) put(mdKey, stickyKey string, acw *acBalancerWrapper) { + ss.mu.Lock() + defer ss.mu.Unlock() + if mdKey != ss.curMDKey { + return + } + // TODO(stickiness): limit the total number of entries. + ss.store[stickyKey] = &stickyStoreEntry{ + acw: acw, + addr: acw.getAddrConn().getCurAddr(), + } +} + +// stickyKey is the key to look up in store. mdKey will be checked against +// curMDKey to avoid races. +func (ss *stickyStore) get(mdKey, stickyKey string) (transport.ClientTransport, bool) { + ss.mu.Lock() + defer ss.mu.Unlock() + if mdKey != ss.curMDKey { + return nil, false + } + entry, ok := ss.store[stickyKey] + if !ok { + return nil, false + } + ac := entry.acw.getAddrConn() + if ac.getCurAddr() != entry.addr { + delete(ss.store, stickyKey) + return nil, false + } + t, ok := ac.getReadyTransport() + if !ok { + delete(ss.store, stickyKey) + return nil, false + } + return t, true +} + +// Get one value from metadata in ctx with key stickinessMDKey. +// +// It returns "", false if stickinessMDKey is an empty string. +func stickyKeyFromContext(ctx context.Context, stickinessMDKey string) (string, bool) { + if stickinessMDKey == "" { + return "", false + } + + md, added, ok := metadata.FromOutgoingContextRaw(ctx) + if !ok { + return "", false + } + + if vv, ok := md[stickinessMDKey]; ok { + if len(vv) > 0 { + return vv[0], true + } + } + + for _, ss := range added { + for i := 0; i < len(ss)-1; i += 2 { + if ss[i] == stickinessMDKey { + return ss[i+1], true + } + } + } + + return "", false +} diff --git a/vendor/google.golang.org/grpc/pickfirst.go b/vendor/google.golang.org/grpc/pickfirst.go new file mode 100644 index 00000000000..bf659d49d2f --- /dev/null +++ b/vendor/google.golang.org/grpc/pickfirst.go @@ -0,0 +1,108 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "golang.org/x/net/context" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" +) + +// PickFirstBalancerName is the name of the pick_first balancer. +const PickFirstBalancerName = "pick_first" + +func newPickfirstBuilder() balancer.Builder { + return &pickfirstBuilder{} +} + +type pickfirstBuilder struct{} + +func (*pickfirstBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { + return &pickfirstBalancer{cc: cc} +} + +func (*pickfirstBuilder) Name() string { + return PickFirstBalancerName +} + +type pickfirstBalancer struct { + cc balancer.ClientConn + sc balancer.SubConn +} + +func (b *pickfirstBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) { + if err != nil { + grpclog.Infof("pickfirstBalancer: HandleResolvedAddrs called with error %v", err) + return + } + if b.sc == nil { + b.sc, err = b.cc.NewSubConn(addrs, balancer.NewSubConnOptions{}) + if err != nil { + grpclog.Errorf("pickfirstBalancer: failed to NewSubConn: %v", err) + return + } + b.cc.UpdateBalancerState(connectivity.Idle, &picker{sc: b.sc}) + b.sc.Connect() + } else { + b.sc.UpdateAddresses(addrs) + b.sc.Connect() + } +} + +func (b *pickfirstBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { + grpclog.Infof("pickfirstBalancer: HandleSubConnStateChange: %p, %v", sc, s) + if b.sc != sc { + grpclog.Infof("pickfirstBalancer: ignored state change because sc is not recognized") + return + } + if s == connectivity.Shutdown { + b.sc = nil + return + } + + switch s { + case connectivity.Ready, connectivity.Idle: + b.cc.UpdateBalancerState(s, &picker{sc: sc}) + case connectivity.Connecting: + b.cc.UpdateBalancerState(s, &picker{err: balancer.ErrNoSubConnAvailable}) + case connectivity.TransientFailure: + b.cc.UpdateBalancerState(s, &picker{err: balancer.ErrTransientFailure}) + } +} + +func (b *pickfirstBalancer) Close() { +} + +type picker struct { + err error + sc balancer.SubConn +} + +func (p *picker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { + if p.err != nil { + return nil, nil, p.err + } + return p.sc, nil, nil +} + +func init() { + balancer.Register(newPickfirstBuilder()) +} diff --git a/vendor/google.golang.org/grpc/proxy.go b/vendor/google.golang.org/grpc/proxy.go new file mode 100644 index 00000000000..2d40236e218 --- /dev/null +++ b/vendor/google.golang.org/grpc/proxy.go @@ -0,0 +1,130 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "bufio" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httputil" + "net/url" + + "golang.org/x/net/context" +) + +var ( + // errDisabled indicates that proxy is disabled for the address. + errDisabled = errors.New("proxy is disabled for the address") + // The following variable will be overwritten in the tests. + httpProxyFromEnvironment = http.ProxyFromEnvironment +) + +func mapAddress(ctx context.Context, address string) (string, error) { + req := &http.Request{ + URL: &url.URL{ + Scheme: "https", + Host: address, + }, + } + url, err := httpProxyFromEnvironment(req) + if err != nil { + return "", err + } + if url == nil { + return "", errDisabled + } + return url.Host, nil +} + +// To read a response from a net.Conn, http.ReadResponse() takes a bufio.Reader. +// It's possible that this reader reads more than what's need for the response and stores +// those bytes in the buffer. +// bufConn wraps the original net.Conn and the bufio.Reader to make sure we don't lose the +// bytes in the buffer. +type bufConn struct { + net.Conn + r io.Reader +} + +func (c *bufConn) Read(b []byte) (int, error) { + return c.r.Read(b) +} + +func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, addr string) (_ net.Conn, err error) { + defer func() { + if err != nil { + conn.Close() + } + }() + + req := (&http.Request{ + Method: http.MethodConnect, + URL: &url.URL{Host: addr}, + Header: map[string][]string{"User-Agent": {grpcUA}}, + }) + + if err := sendHTTPRequest(ctx, req, conn); err != nil { + return nil, fmt.Errorf("failed to write the HTTP request: %v", err) + } + + r := bufio.NewReader(conn) + resp, err := http.ReadResponse(r, req) + if err != nil { + return nil, fmt.Errorf("reading server HTTP response: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + dump, err := httputil.DumpResponse(resp, true) + if err != nil { + return nil, fmt.Errorf("failed to do connect handshake, status code: %s", resp.Status) + } + return nil, fmt.Errorf("failed to do connect handshake, response: %q", dump) + } + + return &bufConn{Conn: conn, r: r}, nil +} + +// newProxyDialer returns a dialer that connects to proxy first if necessary. +// The returned dialer checks if a proxy is necessary, dial to the proxy with the +// provided dialer, does HTTP CONNECT handshake and returns the connection. +func newProxyDialer(dialer func(context.Context, string) (net.Conn, error)) func(context.Context, string) (net.Conn, error) { + return func(ctx context.Context, addr string) (conn net.Conn, err error) { + var skipHandshake bool + newAddr, err := mapAddress(ctx, addr) + if err != nil { + if err != errDisabled { + return nil, err + } + skipHandshake = true + newAddr = addr + } + + conn, err = dialer(ctx, newAddr) + if err != nil { + return + } + if !skipHandshake { + conn, err = doHTTPConnectHandshake(ctx, conn, addr) + } + return + } +} diff --git a/vendor/google.golang.org/grpc/resolver_conn_wrapper.go b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go new file mode 100644 index 00000000000..2c91b038cd8 --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go @@ -0,0 +1,158 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "fmt" + "strings" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" +) + +// ccResolverWrapper is a wrapper on top of cc for resolvers. +// It implements resolver.ClientConnection interface. +type ccResolverWrapper struct { + cc *ClientConn + resolver resolver.Resolver + addrCh chan []resolver.Address + scCh chan string + done chan struct{} +} + +// split2 returns the values from strings.SplitN(s, sep, 2). +// If sep is not found, it returns ("", s, false) instead. +func split2(s, sep string) (string, string, bool) { + spl := strings.SplitN(s, sep, 2) + if len(spl) < 2 { + return "", "", false + } + return spl[0], spl[1], true +} + +// parseTarget splits target into a struct containing scheme, authority and +// endpoint. +// +// If target is not a valid scheme://authority/endpoint, it returns {Endpoint: +// target}. +func parseTarget(target string) (ret resolver.Target) { + var ok bool + ret.Scheme, ret.Endpoint, ok = split2(target, "://") + if !ok { + return resolver.Target{Endpoint: target} + } + ret.Authority, ret.Endpoint, ok = split2(ret.Endpoint, "/") + if !ok { + return resolver.Target{Endpoint: target} + } + return ret +} + +// newCCResolverWrapper parses cc.target for scheme and gets the resolver +// builder for this scheme. It then builds the resolver and starts the +// monitoring goroutine for it. +// +// If withResolverBuilder dial option is set, the specified resolver will be +// used instead. +func newCCResolverWrapper(cc *ClientConn) (*ccResolverWrapper, error) { + rb := cc.dopts.resolverBuilder + if rb == nil { + return nil, fmt.Errorf("could not get resolver for scheme: %q", cc.parsedTarget.Scheme) + } + + ccr := &ccResolverWrapper{ + cc: cc, + addrCh: make(chan []resolver.Address, 1), + scCh: make(chan string, 1), + done: make(chan struct{}), + } + + var err error + ccr.resolver, err = rb.Build(cc.parsedTarget, ccr, resolver.BuildOption{}) + if err != nil { + return nil, err + } + return ccr, nil +} + +func (ccr *ccResolverWrapper) start() { + go ccr.watcher() +} + +// watcher processes address updates and service config updates sequentially. +// Otherwise, we need to resolve possible races between address and service +// config (e.g. they specify different balancer types). +func (ccr *ccResolverWrapper) watcher() { + for { + select { + case <-ccr.done: + return + default: + } + + select { + case addrs := <-ccr.addrCh: + select { + case <-ccr.done: + return + default: + } + grpclog.Infof("ccResolverWrapper: sending new addresses to cc: %v", addrs) + ccr.cc.handleResolvedAddrs(addrs, nil) + case sc := <-ccr.scCh: + select { + case <-ccr.done: + return + default: + } + grpclog.Infof("ccResolverWrapper: got new service config: %v", sc) + ccr.cc.handleServiceConfig(sc) + case <-ccr.done: + return + } + } +} + +func (ccr *ccResolverWrapper) resolveNow(o resolver.ResolveNowOption) { + ccr.resolver.ResolveNow(o) +} + +func (ccr *ccResolverWrapper) close() { + ccr.resolver.Close() + close(ccr.done) +} + +// NewAddress is called by the resolver implemenetion to send addresses to gRPC. +func (ccr *ccResolverWrapper) NewAddress(addrs []resolver.Address) { + select { + case <-ccr.addrCh: + default: + } + ccr.addrCh <- addrs +} + +// NewServiceConfig is called by the resolver implemenetion to send service +// configs to gPRC. +func (ccr *ccResolverWrapper) NewServiceConfig(sc string) { + select { + case <-ccr.scCh: + default: + } + ccr.scCh <- sc +} diff --git a/vendor/google.golang.org/grpc/rpc_util.go b/vendor/google.golang.org/grpc/rpc_util.go index 66d08b5aa2b..f08e646f822 100644 --- a/vendor/google.golang.org/grpc/rpc_util.go +++ b/vendor/google.golang.org/grpc/rpc_util.go @@ -1,33 +1,18 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -41,44 +26,26 @@ import ( "io" "io/ioutil" "math" - "os" + "net/url" + "strings" + "sync" "time" - "github.com/golang/protobuf/proto" "golang.org/x/net/context" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/encoding/proto" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" "google.golang.org/grpc/transport" ) -// Codec defines the interface gRPC uses to encode and decode messages. -type Codec interface { - // Marshal returns the wire format of v. - Marshal(v interface{}) ([]byte, error) - // Unmarshal parses the wire format into v. - Unmarshal(data []byte, v interface{}) error - // String returns the name of the Codec implementation. The returned - // string will be used as part of content type in transmission. - String() string -} - -// protoCodec is a Codec implementation with protobuf. It is the default codec for gRPC. -type protoCodec struct{} - -func (protoCodec) Marshal(v interface{}) ([]byte, error) { - return proto.Marshal(v.(proto.Message)) -} - -func (protoCodec) Unmarshal(data []byte, v interface{}) error { - return proto.Unmarshal(data, v.(proto.Message)) -} - -func (protoCodec) String() string { - return "proto" -} - // Compressor defines the interface gRPC uses to compress a message. +// +// Deprecated: use package encoding. type Compressor interface { // Do compresses p into w. Do(w io.Writer, p []byte) error @@ -86,16 +53,45 @@ type Compressor interface { Type() string } +type gzipCompressor struct { + pool sync.Pool +} + // NewGZIPCompressor creates a Compressor based on GZIP. +// +// Deprecated: use package encoding/gzip. func NewGZIPCompressor() Compressor { - return &gzipCompressor{} + c, _ := NewGZIPCompressorWithLevel(gzip.DefaultCompression) + return c } -type gzipCompressor struct { +// NewGZIPCompressorWithLevel is like NewGZIPCompressor but specifies the gzip compression level instead +// of assuming DefaultCompression. +// +// The error returned will be nil if the level is valid. +// +// Deprecated: use package encoding/gzip. +func NewGZIPCompressorWithLevel(level int) (Compressor, error) { + if level < gzip.DefaultCompression || level > gzip.BestCompression { + return nil, fmt.Errorf("grpc: invalid compression level: %d", level) + } + return &gzipCompressor{ + pool: sync.Pool{ + New: func() interface{} { + w, err := gzip.NewWriterLevel(ioutil.Discard, level) + if err != nil { + panic(err) + } + return w + }, + }, + }, nil } func (c *gzipCompressor) Do(w io.Writer, p []byte) error { - z := gzip.NewWriter(w) + z := c.pool.Get().(*gzip.Writer) + defer c.pool.Put(z) + z.Reset(w) if _, err := z.Write(p); err != nil { return err } @@ -107,6 +103,8 @@ func (c *gzipCompressor) Type() string { } // Decompressor defines the interface gRPC uses to decompress a message. +// +// Deprecated: use package encoding. type Decompressor interface { // Do reads the data from r and uncompress them. Do(r io.Reader) ([]byte, error) @@ -115,19 +113,37 @@ type Decompressor interface { } type gzipDecompressor struct { + pool sync.Pool } // NewGZIPDecompressor creates a Decompressor based on GZIP. +// +// Deprecated: use package encoding/gzip. func NewGZIPDecompressor() Decompressor { return &gzipDecompressor{} } func (d *gzipDecompressor) Do(r io.Reader) ([]byte, error) { - z, err := gzip.NewReader(r) - if err != nil { - return nil, err + var z *gzip.Reader + switch maybeZ := d.pool.Get().(type) { + case nil: + newZ, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + z = newZ + case *gzip.Reader: + z = maybeZ + if err := z.Reset(r); err != nil { + d.pool.Put(z) + return nil, err + } } - defer z.Close() + + defer func() { + z.Close() + d.pool.Put(z) + }() return ioutil.ReadAll(z) } @@ -137,13 +153,20 @@ func (d *gzipDecompressor) Type() string { // callInfo contains all related configuration and information about an RPC. type callInfo struct { - failFast bool - headerMD metadata.MD - trailerMD metadata.MD - traceInfo traceInfo // in trace.go + compressorType string + failFast bool + stream *clientStream + traceInfo traceInfo // in trace.go + maxReceiveMessageSize *int + maxSendMessageSize *int + creds credentials.PerRPCCredentials + contentSubtype string + codec baseCodec } -var defaultCallInfo = callInfo{failFast: true} +func defaultCallInfo() *callInfo { + return &callInfo{failFast: true} +} // CallOption configures a Call before it starts or extracts information from // a Call after it completes. @@ -157,45 +180,241 @@ type CallOption interface { after(*callInfo) } -type beforeCall func(c *callInfo) error - -func (o beforeCall) before(c *callInfo) error { return o(c) } -func (o beforeCall) after(c *callInfo) {} +// EmptyCallOption does not alter the Call configuration. +// It can be embedded in another structure to carry satellite data for use +// by interceptors. +type EmptyCallOption struct{} -type afterCall func(c *callInfo) - -func (o afterCall) before(c *callInfo) error { return nil } -func (o afterCall) after(c *callInfo) { o(c) } +func (EmptyCallOption) before(*callInfo) error { return nil } +func (EmptyCallOption) after(*callInfo) {} // Header returns a CallOptions that retrieves the header metadata // for a unary RPC. func Header(md *metadata.MD) CallOption { - return afterCall(func(c *callInfo) { - *md = c.headerMD - }) + return HeaderCallOption{HeaderAddr: md} +} + +// HeaderCallOption is a CallOption for collecting response header metadata. +// The metadata field will be populated *after* the RPC completes. +// This is an EXPERIMENTAL API. +type HeaderCallOption struct { + HeaderAddr *metadata.MD +} + +func (o HeaderCallOption) before(c *callInfo) error { return nil } +func (o HeaderCallOption) after(c *callInfo) { + if c.stream != nil { + *o.HeaderAddr, _ = c.stream.Header() + } } // Trailer returns a CallOptions that retrieves the trailer metadata // for a unary RPC. func Trailer(md *metadata.MD) CallOption { - return afterCall(func(c *callInfo) { - *md = c.trailerMD - }) + return TrailerCallOption{TrailerAddr: md} +} + +// TrailerCallOption is a CallOption for collecting response trailer metadata. +// The metadata field will be populated *after* the RPC completes. +// This is an EXPERIMENTAL API. +type TrailerCallOption struct { + TrailerAddr *metadata.MD +} + +func (o TrailerCallOption) before(c *callInfo) error { return nil } +func (o TrailerCallOption) after(c *callInfo) { + if c.stream != nil { + *o.TrailerAddr = c.stream.Trailer() + } +} + +// Peer returns a CallOption that retrieves peer information for a unary RPC. +// The peer field will be populated *after* the RPC completes. +func Peer(p *peer.Peer) CallOption { + return PeerCallOption{PeerAddr: p} +} + +// PeerCallOption is a CallOption for collecting the identity of the remote +// peer. The peer field will be populated *after* the RPC completes. +// This is an EXPERIMENTAL API. +type PeerCallOption struct { + PeerAddr *peer.Peer +} + +func (o PeerCallOption) before(c *callInfo) error { return nil } +func (o PeerCallOption) after(c *callInfo) { + if c.stream != nil { + if x, ok := peer.FromContext(c.stream.Context()); ok { + *o.PeerAddr = *x + } + } } // FailFast configures the action to take when an RPC is attempted on broken -// connections or unreachable servers. If failfast is true, the RPC will fail +// connections or unreachable servers. If failFast is true, the RPC will fail // immediately. Otherwise, the RPC client will block the call until a -// connection is available (or the call is canceled or times out) and will retry -// the call if it fails due to a transient error. Please refer to -// https://github.com/grpc/grpc/blob/master/doc/fail_fast.md +// connection is available (or the call is canceled or times out) and will +// retry the call if it fails due to a transient error. gRPC will not retry if +// data was written to the wire unless the server indicates it did not process +// the data. Please refer to +// https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md. +// +// By default, RPCs are "Fail Fast". func FailFast(failFast bool) CallOption { - return beforeCall(func(c *callInfo) error { - c.failFast = failFast - return nil - }) + return FailFastCallOption{FailFast: failFast} +} + +// FailFastCallOption is a CallOption for indicating whether an RPC should fail +// fast or not. +// This is an EXPERIMENTAL API. +type FailFastCallOption struct { + FailFast bool +} + +func (o FailFastCallOption) before(c *callInfo) error { + c.failFast = o.FailFast + return nil +} +func (o FailFastCallOption) after(c *callInfo) {} + +// MaxCallRecvMsgSize returns a CallOption which sets the maximum message size the client can receive. +func MaxCallRecvMsgSize(s int) CallOption { + return MaxRecvMsgSizeCallOption{MaxRecvMsgSize: s} +} + +// MaxRecvMsgSizeCallOption is a CallOption that indicates the maximum message +// size the client can receive. +// This is an EXPERIMENTAL API. +type MaxRecvMsgSizeCallOption struct { + MaxRecvMsgSize int +} + +func (o MaxRecvMsgSizeCallOption) before(c *callInfo) error { + c.maxReceiveMessageSize = &o.MaxRecvMsgSize + return nil +} +func (o MaxRecvMsgSizeCallOption) after(c *callInfo) {} + +// MaxCallSendMsgSize returns a CallOption which sets the maximum message size the client can send. +func MaxCallSendMsgSize(s int) CallOption { + return MaxSendMsgSizeCallOption{MaxSendMsgSize: s} +} + +// MaxSendMsgSizeCallOption is a CallOption that indicates the maximum message +// size the client can send. +// This is an EXPERIMENTAL API. +type MaxSendMsgSizeCallOption struct { + MaxSendMsgSize int +} + +func (o MaxSendMsgSizeCallOption) before(c *callInfo) error { + c.maxSendMessageSize = &o.MaxSendMsgSize + return nil +} +func (o MaxSendMsgSizeCallOption) after(c *callInfo) {} + +// PerRPCCredentials returns a CallOption that sets credentials.PerRPCCredentials +// for a call. +func PerRPCCredentials(creds credentials.PerRPCCredentials) CallOption { + return PerRPCCredsCallOption{Creds: creds} +} + +// PerRPCCredsCallOption is a CallOption that indicates the per-RPC +// credentials to use for the call. +// This is an EXPERIMENTAL API. +type PerRPCCredsCallOption struct { + Creds credentials.PerRPCCredentials +} + +func (o PerRPCCredsCallOption) before(c *callInfo) error { + c.creds = o.Creds + return nil +} +func (o PerRPCCredsCallOption) after(c *callInfo) {} + +// UseCompressor returns a CallOption which sets the compressor used when +// sending the request. If WithCompressor is also set, UseCompressor has +// higher priority. +// +// This API is EXPERIMENTAL. +func UseCompressor(name string) CallOption { + return CompressorCallOption{CompressorType: name} } +// CompressorCallOption is a CallOption that indicates the compressor to use. +// This is an EXPERIMENTAL API. +type CompressorCallOption struct { + CompressorType string +} + +func (o CompressorCallOption) before(c *callInfo) error { + c.compressorType = o.CompressorType + return nil +} +func (o CompressorCallOption) after(c *callInfo) {} + +// CallContentSubtype returns a CallOption that will set the content-subtype +// for a call. For example, if content-subtype is "json", the Content-Type over +// the wire will be "application/grpc+json". The content-subtype is converted +// to lowercase before being included in Content-Type. See Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +// +// If CallCustomCodec is not also used, the content-subtype will be used to +// look up the Codec to use in the registry controlled by RegisterCodec. See +// the documentation on RegisterCodec for details on registration. The lookup +// of content-subtype is case-insensitive. If no such Codec is found, the call +// will result in an error with code codes.Internal. +// +// If CallCustomCodec is also used, that Codec will be used for all request and +// response messages, with the content-subtype set to the given contentSubtype +// here for requests. +func CallContentSubtype(contentSubtype string) CallOption { + return ContentSubtypeCallOption{ContentSubtype: strings.ToLower(contentSubtype)} +} + +// ContentSubtypeCallOption is a CallOption that indicates the content-subtype +// used for marshaling messages. +// This is an EXPERIMENTAL API. +type ContentSubtypeCallOption struct { + ContentSubtype string +} + +func (o ContentSubtypeCallOption) before(c *callInfo) error { + c.contentSubtype = o.ContentSubtype + return nil +} +func (o ContentSubtypeCallOption) after(c *callInfo) {} + +// CallCustomCodec returns a CallOption that will set the given Codec to be +// used for all request and response messages for a call. The result of calling +// String() will be used as the content-subtype in a case-insensitive manner. +// +// See Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. Also see the documentation on RegisterCodec and +// CallContentSubtype for more details on the interaction between Codec and +// content-subtype. +// +// This function is provided for advanced users; prefer to use only +// CallContentSubtype to select a registered codec instead. +func CallCustomCodec(codec Codec) CallOption { + return CustomCodecCallOption{Codec: codec} +} + +// CustomCodecCallOption is a CallOption that indicates the codec used for +// marshaling messages. +// This is an EXPERIMENTAL API. +type CustomCodecCallOption struct { + Codec Codec +} + +func (o CustomCodecCallOption) before(c *callInfo) error { + c.codec = o.Codec + return nil +} +func (o CustomCodecCallOption) after(c *callInfo) {} + // The format of the payload: compressed or not? type payloadFormat uint8 @@ -211,8 +430,8 @@ type parser struct { // error types. r io.Reader - // The header of a gRPC message. Find more detail - // at http://www.grpc.io/docs/guides/wire.html. + // The header of a gRPC message. Find more detail at + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md header [5]byte } @@ -229,8 +448,8 @@ type parser struct { // No other error values or types must be returned, which also means // that the underlying io.Reader must not return an incompatible // error. -func (p *parser) recvMsg(maxMsgSize int) (pf payloadFormat, msg []byte, err error) { - if _, err := io.ReadFull(p.r, p.header[:]); err != nil { +func (p *parser) recvMsg(maxReceiveMessageSize int) (pf payloadFormat, msg []byte, err error) { + if _, err := p.r.Read(p.header[:]); err != nil { return 0, nil, err } @@ -240,13 +459,16 @@ func (p *parser) recvMsg(maxMsgSize int) (pf payloadFormat, msg []byte, err erro if length == 0 { return pf, nil, nil } - if length > uint32(maxMsgSize) { - return 0, nil, Errorf(codes.Internal, "grpc: received message length %d exceeding the max size %d", length, maxMsgSize) + if int64(length) > int64(maxInt) { + return 0, nil, status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max length allowed on current machine (%d vs. %d)", length, maxInt) + } + if int(length) > maxReceiveMessageSize { + return 0, nil, status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", length, maxReceiveMessageSize) } // TODO(bradfitz,zhaoq): garbage. reuse buffer after proto decoding instead // of making it for each message: msg = make([]byte, int(length)) - if _, err := io.ReadFull(p.r, msg); err != nil { + if _, err := p.r.Read(msg); err != nil { if err == io.EOF { err = io.ErrUnexpectedEOF } @@ -255,19 +477,23 @@ func (p *parser) recvMsg(maxMsgSize int) (pf payloadFormat, msg []byte, err erro return pf, msg, nil } -// encode serializes msg and prepends the message header. If msg is nil, it -// generates the message header of 0 message length. -func encode(c Codec, msg interface{}, cp Compressor, cbuf *bytes.Buffer, outPayload *stats.OutPayload) ([]byte, error) { +// encode serializes msg and returns a buffer of message header and a buffer of msg. +// If msg is nil, it generates the message header and an empty msg buffer. +// TODO(ddyihai): eliminate extra Compressor parameter. +func encode(c baseCodec, msg interface{}, cp Compressor, outPayload *stats.OutPayload, compressor encoding.Compressor) ([]byte, []byte, error) { var ( - b []byte - length uint + b []byte + cbuf *bytes.Buffer + ) + const ( + payloadLen = 1 + sizeLen = 4 ) if msg != nil { var err error - // TODO(zhaoq): optimize to reduce memory alloc and copying. b, err = c.Marshal(msg) if err != nil { - return nil, err + return nil, nil, status.Errorf(codes.Internal, "grpc: error while marshaling: %v", err.Error()) } if outPayload != nil { outPayload.Payload = msg @@ -275,80 +501,101 @@ func encode(c Codec, msg interface{}, cp Compressor, cbuf *bytes.Buffer, outPayl outPayload.Data = b outPayload.Length = len(b) } - if cp != nil { - if err := cp.Do(cbuf, b); err != nil { - return nil, err + if compressor != nil || cp != nil { + cbuf = new(bytes.Buffer) + // Has compressor, check Compressor is set by UseCompressor first. + if compressor != nil { + z, _ := compressor.Compress(cbuf) + if _, err := z.Write(b); err != nil { + return nil, nil, status.Errorf(codes.Internal, "grpc: error while compressing: %v", err.Error()) + } + z.Close() + } else { + // If Compressor is not set by UseCompressor, use default Compressor + if err := cp.Do(cbuf, b); err != nil { + return nil, nil, status.Errorf(codes.Internal, "grpc: error while compressing: %v", err.Error()) + } } b = cbuf.Bytes() } - length = uint(len(b)) } - if length > math.MaxUint32 { - return nil, Errorf(codes.InvalidArgument, "grpc: message too large (%d bytes)", length) + if uint(len(b)) > math.MaxUint32 { + return nil, nil, status.Errorf(codes.ResourceExhausted, "grpc: message too large (%d bytes)", len(b)) } - const ( - payloadLen = 1 - sizeLen = 4 - ) - - var buf = make([]byte, payloadLen+sizeLen+len(b)) - - // Write payload format - if cp == nil { - buf[0] = byte(compressionNone) + bufHeader := make([]byte, payloadLen+sizeLen) + if compressor != nil || cp != nil { + bufHeader[0] = byte(compressionMade) } else { - buf[0] = byte(compressionMade) + bufHeader[0] = byte(compressionNone) } - // Write length of b into buf - binary.BigEndian.PutUint32(buf[1:], uint32(length)) - // Copy encoded msg to buf - copy(buf[5:], b) + // Write length of b into buf + binary.BigEndian.PutUint32(bufHeader[payloadLen:], uint32(len(b))) if outPayload != nil { - outPayload.WireLength = len(buf) + outPayload.WireLength = payloadLen + sizeLen + len(b) } - - return buf, nil + return bufHeader, b, nil } -func checkRecvPayload(pf payloadFormat, recvCompress string, dc Decompressor) error { +func checkRecvPayload(pf payloadFormat, recvCompress string, haveCompressor bool) *status.Status { switch pf { case compressionNone: case compressionMade: - if dc == nil || recvCompress != dc.Type() { - return Errorf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", recvCompress) + if recvCompress == "" || recvCompress == encoding.Identity { + return status.New(codes.Internal, "grpc: compressed flag set with identity or empty encoding") + } + if !haveCompressor { + return status.Newf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", recvCompress) } default: - return Errorf(codes.Internal, "grpc: received unexpected payload format %d", pf) + return status.Newf(codes.Internal, "grpc: received unexpected payload format %d", pf) } return nil } -func recv(p *parser, c Codec, s *transport.Stream, dc Decompressor, m interface{}, maxMsgSize int, inPayload *stats.InPayload) error { - pf, d, err := p.recvMsg(maxMsgSize) +// For the two compressor parameters, both should not be set, but if they are, +// dc takes precedence over compressor. +// TODO(dfawley): wrap the old compressor/decompressor using the new API? +func recv(p *parser, c baseCodec, s *transport.Stream, dc Decompressor, m interface{}, maxReceiveMessageSize int, inPayload *stats.InPayload, compressor encoding.Compressor) error { + pf, d, err := p.recvMsg(maxReceiveMessageSize) if err != nil { return err } if inPayload != nil { inPayload.WireLength = len(d) } - if err := checkRecvPayload(pf, s.RecvCompress(), dc); err != nil { - return err + + if st := checkRecvPayload(pf, s.RecvCompress(), compressor != nil || dc != nil); st != nil { + return st.Err() } + if pf == compressionMade { - d, err = dc.Do(bytes.NewReader(d)) - if err != nil { - return Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + // To match legacy behavior, if the decompressor is set by WithDecompressor or RPCDecompressor, + // use this decompressor as the default. + if dc != nil { + d, err = dc.Do(bytes.NewReader(d)) + if err != nil { + return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + } + } else { + dcReader, err := compressor.Decompress(bytes.NewReader(d)) + if err != nil { + return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + } + d, err = ioutil.ReadAll(dcReader) + if err != nil { + return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + } } } - if len(d) > maxMsgSize { + if len(d) > maxReceiveMessageSize { // TODO: Revisit the error code. Currently keep it consistent with java // implementation. - return Errorf(codes.Internal, "grpc: received a message of %d bytes exceeding %d limit", len(d), maxMsgSize) + return status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", len(d), maxReceiveMessageSize) } if err := c.Unmarshal(d, m); err != nil { - return Errorf(codes.Internal, "grpc: failed to unmarshal the received message %v", err) + return status.Errorf(codes.Internal, "grpc: failed to unmarshal the received message %v", err) } if inPayload != nil { inPayload.RecvTime = time.Now() @@ -360,122 +607,121 @@ func recv(p *parser, c Codec, s *transport.Stream, dc Decompressor, m interface{ return nil } -// rpcError defines the status from an RPC. -type rpcError struct { - code codes.Code - desc string +type rpcInfo struct { + failfast bool } -func (e *rpcError) Error() string { - return fmt.Sprintf("rpc error: code = %d desc = %s", e.code, e.desc) +type rpcInfoContextKey struct{} + +func newContextWithRPCInfo(ctx context.Context, failfast bool) context.Context { + return context.WithValue(ctx, rpcInfoContextKey{}, &rpcInfo{failfast: failfast}) +} + +func rpcInfoFromContext(ctx context.Context) (s *rpcInfo, ok bool) { + s, ok = ctx.Value(rpcInfoContextKey{}).(*rpcInfo) + return } // Code returns the error code for err if it was produced by the rpc system. // Otherwise, it returns codes.Unknown. +// +// Deprecated: use status.FromError and Code method instead. func Code(err error) codes.Code { - if err == nil { - return codes.OK - } - if e, ok := err.(*rpcError); ok { - return e.code + if s, ok := status.FromError(err); ok { + return s.Code() } return codes.Unknown } // ErrorDesc returns the error description of err if it was produced by the rpc system. // Otherwise, it returns err.Error() or empty string when err is nil. +// +// Deprecated: use status.FromError and Message method instead. func ErrorDesc(err error) string { - if err == nil { - return "" - } - if e, ok := err.(*rpcError); ok { - return e.desc + if s, ok := status.FromError(err); ok { + return s.Message() } return err.Error() } // Errorf returns an error containing an error code and a description; // Errorf returns nil if c is OK. +// +// Deprecated: use status.Errorf instead. func Errorf(c codes.Code, format string, a ...interface{}) error { - if c == codes.OK { + return status.Errorf(c, format, a...) +} + +// setCallInfoCodec should only be called after CallOptions have been applied. +func setCallInfoCodec(c *callInfo) error { + if c.codec != nil { + // codec was already set by a CallOption; use it. return nil } - return &rpcError{ - code: c, - desc: fmt.Sprintf(format, a...), + + if c.contentSubtype == "" { + // No codec specified in CallOptions; use proto by default. + c.codec = encoding.GetCodec(proto.Name) + return nil } + + // c.contentSubtype is already lowercased in CallContentSubtype + c.codec = encoding.GetCodec(c.contentSubtype) + if c.codec == nil { + return status.Errorf(codes.Internal, "no codec registered for content-subtype %s", c.contentSubtype) + } + return nil } -// toRPCErr converts an error into a rpcError. -func toRPCErr(err error) error { - switch e := err.(type) { - case *rpcError: - return err - case transport.StreamError: - return &rpcError{ - code: e.Code, - desc: e.Desc, +// parseDialTarget returns the network and address to pass to dialer +func parseDialTarget(target string) (net string, addr string) { + net = "tcp" + + m1 := strings.Index(target, ":") + m2 := strings.Index(target, ":/") + + // handle unix:addr which will fail with url.Parse + if m1 >= 0 && m2 < 0 { + if n := target[0:m1]; n == "unix" { + net = n + addr = target[m1+1:] + return net, addr } - case transport.ConnectionError: - return &rpcError{ - code: codes.Internal, - desc: e.Desc, + } + if m2 >= 0 { + t, err := url.Parse(target) + if err != nil { + return net, target } - default: - switch err { - case context.DeadlineExceeded: - return &rpcError{ - code: codes.DeadlineExceeded, - desc: err.Error(), - } - case context.Canceled: - return &rpcError{ - code: codes.Canceled, - desc: err.Error(), - } - case ErrClientConnClosing: - return &rpcError{ - code: codes.FailedPrecondition, - desc: err.Error(), + scheme := t.Scheme + addr = t.Path + if scheme == "unix" { + net = scheme + if addr == "" { + addr = t.Host } + return net, addr } - } - return Errorf(codes.Unknown, "%v", err) -} -// convertCode converts a standard Go error into its canonical code. Note that -// this is only used to translate the error returned by the server applications. -func convertCode(err error) codes.Code { - switch err { - case nil: - return codes.OK - case io.EOF: - return codes.OutOfRange - case io.ErrClosedPipe, io.ErrNoProgress, io.ErrShortBuffer, io.ErrShortWrite, io.ErrUnexpectedEOF: - return codes.FailedPrecondition - case os.ErrInvalid: - return codes.InvalidArgument - case context.Canceled: - return codes.Canceled - case context.DeadlineExceeded: - return codes.DeadlineExceeded - } - switch { - case os.IsExist(err): - return codes.AlreadyExists - case os.IsNotExist(err): - return codes.NotFound - case os.IsPermission(err): - return codes.PermissionDenied - } - return codes.Unknown + return net, target } -// SupportPackageIsVersion4 is referenced from generated protocol buffer files -// to assert that that code is compatible with this version of the grpc package. +// The SupportPackageIsVersion variables are referenced from generated protocol +// buffer files to ensure compatibility with the gRPC version used. The latest +// support package version is 5. +// +// Older versions are kept for compatibility. They may be removed if +// compatibility cannot be maintained. // -// This constant may be renamed in the future if a change in the generated code -// requires a synchronised update of grpc-go and protoc-gen-go. This constant -// should not be referenced from any other code. -const SupportPackageIsVersion4 = true +// These constants should not be referenced from any other code. +const ( + SupportPackageIsVersion3 = true + SupportPackageIsVersion4 = true + SupportPackageIsVersion5 = true +) + +// Version is the current grpc version. +const Version = "1.12.0-dev" + +const grpcUA = "grpc-go/" + Version diff --git a/vendor/google.golang.org/grpc/server.go b/vendor/google.golang.org/grpc/server.go index 22aa33bfe4a..4969331cb3d 100644 --- a/vendor/google.golang.org/grpc/server.go +++ b/vendor/google.golang.org/grpc/server.go @@ -1,33 +1,18 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -38,6 +23,7 @@ import ( "errors" "fmt" "io" + "math" "net" "net/http" "reflect" @@ -46,19 +32,32 @@ import ( "sync" "time" + "io/ioutil" + "golang.org/x/net/context" "golang.org/x/net/http2" "golang.org/x/net/trace" + + "google.golang.org/grpc/channelz" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/encoding/proto" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal" + "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" "google.golang.org/grpc/tap" "google.golang.org/grpc/transport" ) +const ( + defaultServerMaxReceiveMessageSize = 1024 * 1024 * 4 + defaultServerMaxSendMessageSize = math.MaxInt32 +) + type methodHandler func(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor UnaryServerInterceptor) (interface{}, error) // MethodDesc represents an RPC service's method specification. @@ -94,60 +93,159 @@ type Server struct { mu sync.Mutex // guards following lis map[net.Listener]bool conns map[io.Closer]bool + serve bool drain bool - ctx context.Context - cancel context.CancelFunc - // A CondVar to let GracefulStop() blocks until all the pending RPCs are finished - // and all the transport goes away. - cv *sync.Cond + cv *sync.Cond // signaled when connections close for GracefulStop m map[string]*service // service name -> service info events trace.EventLog + + quit chan struct{} + done chan struct{} + quitOnce sync.Once + doneOnce sync.Once + channelzRemoveOnce sync.Once + serveWG sync.WaitGroup // counts active Serve goroutines for GracefulStop + + channelzID int64 // channelz unique identification number + czmu sync.RWMutex + callsStarted int64 + callsFailed int64 + callsSucceeded int64 + lastCallStartedTime time.Time } type options struct { - creds credentials.TransportCredentials - codec Codec - cp Compressor - dc Decompressor - maxMsgSize int - unaryInt UnaryServerInterceptor - streamInt StreamServerInterceptor - inTapHandle tap.ServerInHandle - maxConcurrentStreams uint32 - useHandlerImpl bool // use http.Handler-based server + creds credentials.TransportCredentials + codec baseCodec + cp Compressor + dc Decompressor + unaryInt UnaryServerInterceptor + streamInt StreamServerInterceptor + inTapHandle tap.ServerInHandle + statsHandler stats.Handler + maxConcurrentStreams uint32 + maxReceiveMessageSize int + maxSendMessageSize int + useHandlerImpl bool // use http.Handler-based server + unknownStreamDesc *StreamDesc + keepaliveParams keepalive.ServerParameters + keepalivePolicy keepalive.EnforcementPolicy + initialWindowSize int32 + initialConnWindowSize int32 + writeBufferSize int + readBufferSize int + connectionTimeout time.Duration } -var defaultMaxMsgSize = 1024 * 1024 * 4 // use 4MB as the default message size limit +var defaultServerOptions = options{ + maxReceiveMessageSize: defaultServerMaxReceiveMessageSize, + maxSendMessageSize: defaultServerMaxSendMessageSize, + connectionTimeout: 120 * time.Second, +} -// A ServerOption sets options. +// A ServerOption sets options such as credentials, codec and keepalive parameters, etc. type ServerOption func(*options) +// WriteBufferSize lets you set the size of write buffer, this determines how much data can be batched +// before doing a write on the wire. +func WriteBufferSize(s int) ServerOption { + return func(o *options) { + o.writeBufferSize = s + } +} + +// ReadBufferSize lets you set the size of read buffer, this determines how much data can be read at most +// for one read syscall. +func ReadBufferSize(s int) ServerOption { + return func(o *options) { + o.readBufferSize = s + } +} + +// InitialWindowSize returns a ServerOption that sets window size for stream. +// The lower bound for window size is 64K and any value smaller than that will be ignored. +func InitialWindowSize(s int32) ServerOption { + return func(o *options) { + o.initialWindowSize = s + } +} + +// InitialConnWindowSize returns a ServerOption that sets window size for a connection. +// The lower bound for window size is 64K and any value smaller than that will be ignored. +func InitialConnWindowSize(s int32) ServerOption { + return func(o *options) { + o.initialConnWindowSize = s + } +} + +// KeepaliveParams returns a ServerOption that sets keepalive and max-age parameters for the server. +func KeepaliveParams(kp keepalive.ServerParameters) ServerOption { + return func(o *options) { + o.keepaliveParams = kp + } +} + +// KeepaliveEnforcementPolicy returns a ServerOption that sets keepalive enforcement policy for the server. +func KeepaliveEnforcementPolicy(kep keepalive.EnforcementPolicy) ServerOption { + return func(o *options) { + o.keepalivePolicy = kep + } +} + // CustomCodec returns a ServerOption that sets a codec for message marshaling and unmarshaling. +// +// This will override any lookups by content-subtype for Codecs registered with RegisterCodec. func CustomCodec(codec Codec) ServerOption { return func(o *options) { o.codec = codec } } -// RPCCompressor returns a ServerOption that sets a compressor for outbound messages. +// RPCCompressor returns a ServerOption that sets a compressor for outbound +// messages. For backward compatibility, all outbound messages will be sent +// using this compressor, regardless of incoming message compression. By +// default, server messages will be sent using the same compressor with which +// request messages were sent. +// +// Deprecated: use encoding.RegisterCompressor instead. func RPCCompressor(cp Compressor) ServerOption { return func(o *options) { o.cp = cp } } -// RPCDecompressor returns a ServerOption that sets a decompressor for inbound messages. +// RPCDecompressor returns a ServerOption that sets a decompressor for inbound +// messages. It has higher priority than decompressors registered via +// encoding.RegisterCompressor. +// +// Deprecated: use encoding.RegisterCompressor instead. func RPCDecompressor(dc Decompressor) ServerOption { return func(o *options) { o.dc = dc } } -// MaxMsgSize returns a ServerOption to set the max message size in bytes for inbound mesages. -// If this is not set, gRPC uses the default 4MB. +// MaxMsgSize returns a ServerOption to set the max message size in bytes the server can receive. +// If this is not set, gRPC uses the default limit. +// +// Deprecated: use MaxRecvMsgSize instead. func MaxMsgSize(m int) ServerOption { + return MaxRecvMsgSize(m) +} + +// MaxRecvMsgSize returns a ServerOption to set the max message size in bytes the server can receive. +// If this is not set, gRPC uses the default 4MB. +func MaxRecvMsgSize(m int) ServerOption { + return func(o *options) { + o.maxReceiveMessageSize = m + } +} + +// MaxSendMsgSize returns a ServerOption to set the max message size in bytes the server can send. +// If this is not set, gRPC uses the default 4MB. +func MaxSendMsgSize(m int) ServerOption { return func(o *options) { - o.maxMsgSize = m + o.maxSendMessageSize = m } } @@ -172,7 +270,7 @@ func Creds(c credentials.TransportCredentials) ServerOption { func UnaryInterceptor(i UnaryServerInterceptor) ServerOption { return func(o *options) { if o.unaryInt != nil { - panic("The unary server interceptor has been set.") + panic("The unary server interceptor was already set and may not be reset.") } o.unaryInt = i } @@ -183,7 +281,7 @@ func UnaryInterceptor(i UnaryServerInterceptor) ServerOption { func StreamInterceptor(i StreamServerInterceptor) ServerOption { return func(o *options) { if o.streamInt != nil { - panic("The stream server interceptor has been set.") + panic("The stream server interceptor was already set and may not be reset.") } o.streamInt = i } @@ -194,36 +292,73 @@ func StreamInterceptor(i StreamServerInterceptor) ServerOption { func InTapHandle(h tap.ServerInHandle) ServerOption { return func(o *options) { if o.inTapHandle != nil { - panic("The tap handle has been set.") + panic("The tap handle was already set and may not be reset.") } o.inTapHandle = h } } +// StatsHandler returns a ServerOption that sets the stats handler for the server. +func StatsHandler(h stats.Handler) ServerOption { + return func(o *options) { + o.statsHandler = h + } +} + +// UnknownServiceHandler returns a ServerOption that allows for adding a custom +// unknown service handler. The provided method is a bidi-streaming RPC service +// handler that will be invoked instead of returning the "unimplemented" gRPC +// error whenever a request is received for an unregistered service or method. +// The handling function has full access to the Context of the request and the +// stream, and the invocation bypasses interceptors. +func UnknownServiceHandler(streamHandler StreamHandler) ServerOption { + return func(o *options) { + o.unknownStreamDesc = &StreamDesc{ + StreamName: "unknown_service_handler", + Handler: streamHandler, + // We need to assume that the users of the streamHandler will want to use both. + ClientStreams: true, + ServerStreams: true, + } + } +} + +// ConnectionTimeout returns a ServerOption that sets the timeout for +// connection establishment (up to and including HTTP/2 handshaking) for all +// new connections. If this is not set, the default is 120 seconds. A zero or +// negative value will result in an immediate timeout. +// +// This API is EXPERIMENTAL. +func ConnectionTimeout(d time.Duration) ServerOption { + return func(o *options) { + o.connectionTimeout = d + } +} + // NewServer creates a gRPC server which has no service registered and has not // started to accept requests yet. func NewServer(opt ...ServerOption) *Server { - var opts options - opts.maxMsgSize = defaultMaxMsgSize + opts := defaultServerOptions for _, o := range opt { o(&opts) } - if opts.codec == nil { - // Set the default codec. - opts.codec = protoCodec{} - } s := &Server{ lis: make(map[net.Listener]bool), opts: opts, conns: make(map[io.Closer]bool), m: make(map[string]*service), + quit: make(chan struct{}), + done: make(chan struct{}), } s.cv = sync.NewCond(&s.mu) - s.ctx, s.cancel = context.WithCancel(context.Background()) if EnableTracing { _, file, line, _ := runtime.Caller(1) s.events = trace.NewEventLog("grpc.Server", fmt.Sprintf("%s:%d", file, line)) } + + if channelz.IsOn() { + s.channelzID = channelz.RegisterServer(s, "") + } return s } @@ -243,8 +378,8 @@ func (s *Server) errorf(format string, a ...interface{}) { } } -// RegisterService register a service and its implementation to the gRPC -// server. Called from the IDL generated code. This must be called before +// RegisterService registers a service and its implementation to the gRPC +// server. It is called from the IDL generated code. This must be called before // invoking Serve. func (s *Server) RegisterService(sd *ServiceDesc, ss interface{}) { ht := reflect.TypeOf(sd.HandlerType).Elem() @@ -259,6 +394,9 @@ func (s *Server) register(sd *ServiceDesc, ss interface{}) { s.mu.Lock() defer s.mu.Unlock() s.printf("RegisterService(%q)", sd.ServiceName) + if s.serve { + grpclog.Fatalf("grpc: Server.RegisterService after Server.Serve for %q", sd.ServiceName) + } if _, ok := s.m[sd.ServiceName]; ok { grpclog.Fatalf("grpc: Server.RegisterService found duplicate service registration for %q", sd.ServiceName) } @@ -289,7 +427,7 @@ type MethodInfo struct { IsServerStream bool } -// ServiceInfo contains unary RPC method info, streaming RPC methid info and metadata for a service. +// ServiceInfo contains unary RPC method info, streaming RPC method info and metadata for a service. type ServiceInfo struct { Methods []MethodInfo // Metadata is the metadata specified in ServiceDesc when registering service. @@ -325,11 +463,9 @@ func (s *Server) GetServiceInfo() map[string]ServiceInfo { return ret } -var ( - // ErrServerStopped indicates that the operation is now illegal because of - // the server being stopped. - ErrServerStopped = errors.New("grpc: the server has been stopped") -) +// ErrServerStopped indicates that the operation is now illegal because of +// the server being stopped. +var ErrServerStopped = errors.New("grpc: the server has been stopped") func (s *Server) useTransportAuthenticator(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { if s.opts.creds == nil { @@ -338,26 +474,66 @@ func (s *Server) useTransportAuthenticator(rawConn net.Conn) (net.Conn, credenti return s.opts.creds.ServerHandshake(rawConn) } +type listenSocket struct { + net.Listener + channelzID int64 +} + +func (l *listenSocket) ChannelzMetric() *channelz.SocketInternalMetric { + return &channelz.SocketInternalMetric{ + LocalAddr: l.Listener.Addr(), + } +} + +func (l *listenSocket) Close() error { + err := l.Listener.Close() + if channelz.IsOn() { + channelz.RemoveEntry(l.channelzID) + } + return err +} + // Serve accepts incoming connections on the listener lis, creating a new // ServerTransport and service goroutine for each. The service goroutines // read gRPC requests and then call the registered handlers to reply to them. // Serve returns when lis.Accept fails with fatal errors. lis will be closed when // this method returns. +// Serve will return a non-nil error unless Stop or GracefulStop is called. func (s *Server) Serve(lis net.Listener) error { s.mu.Lock() s.printf("serving") + s.serve = true if s.lis == nil { + // Serve called after Stop or GracefulStop. s.mu.Unlock() lis.Close() return ErrServerStopped } - s.lis[lis] = true + + s.serveWG.Add(1) + defer func() { + s.serveWG.Done() + select { + // Stop or GracefulStop called; block until done and return nil. + case <-s.quit: + <-s.done + default: + } + }() + + ls := &listenSocket{Listener: lis} + s.lis[ls] = true + + if channelz.IsOn() { + ls.channelzID = channelz.RegisterListenSocket(ls, s.channelzID, "") + } s.mu.Unlock() + defer func() { s.mu.Lock() - if s.lis != nil && s.lis[lis] { - lis.Close() - delete(s.lis, lis) + if s.lis != nil && s.lis[ls] { + ls.Close() + delete(s.lis, ls) } s.mu.Unlock() }() @@ -381,37 +557,55 @@ func (s *Server) Serve(lis net.Listener) error { s.mu.Lock() s.printf("Accept error: %v; retrying in %v", err, tempDelay) s.mu.Unlock() + timer := time.NewTimer(tempDelay) select { - case <-time.After(tempDelay): - case <-s.ctx.Done(): + case <-timer.C: + case <-s.quit: + timer.Stop() + return nil } continue } s.mu.Lock() s.printf("done serving; Accept = %v", err) s.mu.Unlock() + + select { + case <-s.quit: + return nil + default: + } return err } tempDelay = 0 - // Start a new goroutine to deal with rawConn - // so we don't stall this Accept loop goroutine. - go s.handleRawConn(rawConn) + // Start a new goroutine to deal with rawConn so we don't stall this Accept + // loop goroutine. + // + // Make sure we account for the goroutine so GracefulStop doesn't nil out + // s.conns before this conn can be added. + s.serveWG.Add(1) + go func() { + s.handleRawConn(rawConn) + s.serveWG.Done() + }() } } -// handleRawConn is run in its own goroutine and handles a just-accepted -// connection that has not had any I/O performed on it yet. +// handleRawConn forks a goroutine to handle a just-accepted connection that +// has not had any I/O performed on it yet. func (s *Server) handleRawConn(rawConn net.Conn) { + rawConn.SetDeadline(time.Now().Add(s.opts.connectionTimeout)) conn, authInfo, err := s.useTransportAuthenticator(rawConn) if err != nil { s.mu.Lock() s.errorf("ServerHandshake(%q) failed: %v", rawConn.RemoteAddr(), err) s.mu.Unlock() - grpclog.Printf("grpc: Server.Serve failed to complete security handshake from %q: %v", rawConn.RemoteAddr(), err) - // If serverHandShake returns ErrConnDispatched, keep rawConn open. + grpclog.Warningf("grpc: Server.Serve failed to complete security handshake from %q: %v", rawConn.RemoteAddr(), err) + // If serverHandshake returns ErrConnDispatched, keep rawConn open. if err != credentials.ErrConnDispatched { rawConn.Close() } + rawConn.SetDeadline(time.Time{}) return } @@ -423,23 +617,45 @@ func (s *Server) handleRawConn(rawConn net.Conn) { } s.mu.Unlock() + var serve func() + c := conn.(io.Closer) if s.opts.useHandlerImpl { - s.serveUsingHandler(conn) + serve = func() { s.serveUsingHandler(conn) } } else { - s.serveHTTP2Transport(conn, authInfo) + // Finish handshaking (HTTP2) + st := s.newHTTP2Transport(conn, authInfo) + if st == nil { + return + } + c = st + serve = func() { s.serveStreams(st) } } + + rawConn.SetDeadline(time.Time{}) + if !s.addConn(c) { + return + } + go func() { + serve() + s.removeConn(c) + }() } -// serveHTTP2Transport sets up a http/2 transport (using the -// gRPC http2 server transport in transport/http2_server.go) and -// serves streams on it. -// This is run in its own goroutine (it does network I/O in -// transport.NewServerTransport). -func (s *Server) serveHTTP2Transport(c net.Conn, authInfo credentials.AuthInfo) { +// newHTTP2Transport sets up a http/2 transport (using the +// gRPC http2 server transport in transport/http2_server.go). +func (s *Server) newHTTP2Transport(c net.Conn, authInfo credentials.AuthInfo) transport.ServerTransport { config := &transport.ServerConfig{ - MaxStreams: s.opts.maxConcurrentStreams, - AuthInfo: authInfo, - InTapHandle: s.opts.inTapHandle, + MaxStreams: s.opts.maxConcurrentStreams, + AuthInfo: authInfo, + InTapHandle: s.opts.inTapHandle, + StatsHandler: s.opts.statsHandler, + KeepaliveParams: s.opts.keepaliveParams, + KeepalivePolicy: s.opts.keepalivePolicy, + InitialWindowSize: s.opts.initialWindowSize, + InitialConnWindowSize: s.opts.initialConnWindowSize, + WriteBufferSize: s.opts.writeBufferSize, + ReadBufferSize: s.opts.readBufferSize, + ChannelzParentID: s.channelzID, } st, err := transport.NewServerTransport("http2", c, config) if err != nil { @@ -447,18 +663,14 @@ func (s *Server) serveHTTP2Transport(c net.Conn, authInfo credentials.AuthInfo) s.errorf("NewServerTransport(%q) failed: %v", c.RemoteAddr(), err) s.mu.Unlock() c.Close() - grpclog.Println("grpc: Server.Serve failed to create ServerTransport: ", err) - return - } - if !s.addConn(st) { - st.Close() - return + grpclog.Warningln("grpc: Server.Serve failed to create ServerTransport: ", err) + return nil } - s.serveStreams(st) + + return st } func (s *Server) serveStreams(st transport.ServerTransport) { - defer s.removeConn(st) defer st.Close() var wg sync.WaitGroup st.HandleStreams(func(stream *transport.Stream) { @@ -492,11 +704,6 @@ var _ http.Handler = (*Server)(nil) // // conn is the *tls.Conn that's already been authenticated. func (s *Server) serveUsingHandler(conn net.Conn) { - if !s.addConn(conn) { - conn.Close() - return - } - defer s.removeConn(conn) h2s := &http2.Server{ MaxConcurrentStreams: s.opts.maxConcurrentStreams, } @@ -505,14 +712,37 @@ func (s *Server) serveUsingHandler(conn net.Conn) { }) } +// ServeHTTP implements the Go standard library's http.Handler +// interface by responding to the gRPC request r, by looking up +// the requested gRPC method in the gRPC server s. +// +// The provided HTTP request must have arrived on an HTTP/2 +// connection. When using the Go standard library's server, +// practically this means that the Request must also have arrived +// over TLS. +// +// To share one port (such as 443 for https) between gRPC and an +// existing http.Handler, use a root http.Handler such as: +// +// if r.ProtoMajor == 2 && strings.HasPrefix( +// r.Header.Get("Content-Type"), "application/grpc") { +// grpcServer.ServeHTTP(w, r) +// } else { +// yourMux.ServeHTTP(w, r) +// } +// +// Note that ServeHTTP uses Go's HTTP/2 server implementation which is totally +// separate from grpc-go's HTTP/2 server. Performance and features may vary +// between the two paths. ServeHTTP does not support some gRPC features +// available through grpc-go's HTTP/2 server, and it is currently EXPERIMENTAL +// and subject to change. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - st, err := transport.NewServerHandlerTransport(w, r) + st, err := transport.NewServerHandlerTransport(w, r, s.opts.statsHandler) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } if !s.addConn(st) { - st.Close() return } defer s.removeConn(st) @@ -542,9 +772,15 @@ func (s *Server) traceInfo(st transport.ServerTransport, stream *transport.Strea func (s *Server) addConn(c io.Closer) bool { s.mu.Lock() defer s.mu.Unlock() - if s.conns == nil || s.drain { + if s.conns == nil { + c.Close() return false } + if s.drain { + // Transport added after we drained our existing conns: drain it + // immediately. + c.(transport.ServerTransport).Drain() + } s.conns[c] = true return true } @@ -558,54 +794,90 @@ func (s *Server) removeConn(c io.Closer) { } } -func (s *Server) sendResponse(t transport.ServerTransport, stream *transport.Stream, msg interface{}, cp Compressor, opts *transport.Options) error { +// ChannelzMetric returns ServerInternalMetric of current server. +// This is an EXPERIMENTAL API. +func (s *Server) ChannelzMetric() *channelz.ServerInternalMetric { + s.czmu.RLock() + defer s.czmu.RUnlock() + return &channelz.ServerInternalMetric{ + CallsStarted: s.callsStarted, + CallsSucceeded: s.callsSucceeded, + CallsFailed: s.callsFailed, + LastCallStartedTimestamp: s.lastCallStartedTime, + } +} + +func (s *Server) incrCallsStarted() { + s.czmu.Lock() + s.callsStarted++ + s.lastCallStartedTime = time.Now() + s.czmu.Unlock() +} + +func (s *Server) incrCallsSucceeded() { + s.czmu.Lock() + s.callsSucceeded++ + s.czmu.Unlock() +} + +func (s *Server) incrCallsFailed() { + s.czmu.Lock() + s.callsFailed++ + s.czmu.Unlock() +} + +func (s *Server) sendResponse(t transport.ServerTransport, stream *transport.Stream, msg interface{}, cp Compressor, opts *transport.Options, comp encoding.Compressor) error { var ( - cbuf *bytes.Buffer outPayload *stats.OutPayload ) - if cp != nil { - cbuf = new(bytes.Buffer) - } - if stats.On() { + if s.opts.statsHandler != nil { outPayload = &stats.OutPayload{} } - p, err := encode(s.opts.codec, msg, cp, cbuf, outPayload) + hdr, data, err := encode(s.getCodec(stream.ContentSubtype()), msg, cp, outPayload, comp) if err != nil { - // This typically indicates a fatal issue (e.g., memory - // corruption or hardware faults) the application program - // cannot handle. - // - // TODO(zhaoq): There exist other options also such as only closing the - // faulty stream locally and remotely (Other streams can keep going). Find - // the optimal option. - grpclog.Fatalf("grpc: Server failed to encode response %v", err) + grpclog.Errorln("grpc: server failed to encode response: ", err) + return err + } + if len(data) > s.opts.maxSendMessageSize { + return status.Errorf(codes.ResourceExhausted, "grpc: trying to send message larger than max (%d vs. %d)", len(data), s.opts.maxSendMessageSize) } - err = t.Write(stream, p, opts) + err = t.Write(stream, hdr, data, opts) if err == nil && outPayload != nil { outPayload.SentTime = time.Now() - stats.HandleRPC(stream.Context(), outPayload) + s.opts.statsHandler.HandleRPC(stream.Context(), outPayload) } return err } func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, md *MethodDesc, trInfo *traceInfo) (err error) { - if stats.On() { + if channelz.IsOn() { + s.incrCallsStarted() + defer func() { + if err != nil && err != io.EOF { + s.incrCallsFailed() + } else { + s.incrCallsSucceeded() + } + }() + } + sh := s.opts.statsHandler + if sh != nil { + beginTime := time.Now() begin := &stats.Begin{ - BeginTime: time.Now(), + BeginTime: beginTime, } - stats.HandleRPC(stream.Context(), begin) - } - defer func() { - if stats.On() { + sh.HandleRPC(stream.Context(), begin) + defer func() { end := &stats.End{ - EndTime: time.Now(), + BeginTime: beginTime, + EndTime: time.Now(), } if err != nil && err != io.EOF { end.Error = toRPCErr(err) } - stats.HandleRPC(stream.Context(), end) - } - }() + sh.HandleRPC(stream.Context(), end) + }() + } if trInfo != nil { defer trInfo.tr.Finish() trInfo.firstLine.client = false @@ -617,177 +889,255 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } }() } + + // comp and cp are used for compression. decomp and dc are used for + // decompression. If comp and decomp are both set, they are the same; + // however they are kept separate to ensure that at most one of the + // compressor/decompressor variable pairs are set for use later. + var comp, decomp encoding.Compressor + var cp Compressor + var dc Decompressor + + // If dc is set and matches the stream's compression, use it. Otherwise, try + // to find a matching registered compressor for decomp. + if rc := stream.RecvCompress(); s.opts.dc != nil && s.opts.dc.Type() == rc { + dc = s.opts.dc + } else if rc != "" && rc != encoding.Identity { + decomp = encoding.GetCompressor(rc) + if decomp == nil { + st := status.Newf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", rc) + t.WriteStatus(stream, st) + return st.Err() + } + } + + // If cp is set, use it. Otherwise, attempt to compress the response using + // the incoming message compression method. + // + // NOTE: this needs to be ahead of all handling, https://github.com/grpc/grpc-go/issues/686. if s.opts.cp != nil { - // NOTE: this needs to be ahead of all handling, https://github.com/grpc/grpc-go/issues/686. - stream.SetSendCompress(s.opts.cp.Type()) + cp = s.opts.cp + stream.SetSendCompress(cp.Type()) + } else if rc := stream.RecvCompress(); rc != "" && rc != encoding.Identity { + // Legacy compressor not specified; attempt to respond with same encoding. + comp = encoding.GetCompressor(rc) + if comp != nil { + stream.SetSendCompress(rc) + } } + p := &parser{r: stream} - for { - pf, req, err := p.recvMsg(s.opts.maxMsgSize) - if err == io.EOF { - // The entire stream is done (for unary RPC only). - return err - } - if err == io.ErrUnexpectedEOF { - err = Errorf(codes.Internal, io.ErrUnexpectedEOF.Error()) - } - if err != nil { - switch err := err.(type) { - case *rpcError: - if e := t.WriteStatus(stream, err.code, err.desc); e != nil { - grpclog.Printf("grpc: Server.processUnaryRPC failed to write status %v", e) - } + pf, req, err := p.recvMsg(s.opts.maxReceiveMessageSize) + if err == io.EOF { + // The entire stream is done (for unary RPC only). + return err + } + if err == io.ErrUnexpectedEOF { + err = status.Errorf(codes.Internal, io.ErrUnexpectedEOF.Error()) + } + if err != nil { + if st, ok := status.FromError(err); ok { + if e := t.WriteStatus(stream, st); e != nil { + grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status %v", e) + } + } else { + switch st := err.(type) { case transport.ConnectionError: // Nothing to do here. case transport.StreamError: - if e := t.WriteStatus(stream, err.Code, err.Desc); e != nil { - grpclog.Printf("grpc: Server.processUnaryRPC failed to write status %v", e) + if e := t.WriteStatus(stream, status.New(st.Code, st.Desc)); e != nil { + grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status %v", e) } default: - panic(fmt.Sprintf("grpc: Unexpected error (%T) from recvMsg: %v", err, err)) + panic(fmt.Sprintf("grpc: Unexpected error (%T) from recvMsg: %v", st, st)) } - return err } - - if err := checkRecvPayload(pf, stream.RecvCompress(), s.opts.dc); err != nil { - switch err := err.(type) { - case *rpcError: - if e := t.WriteStatus(stream, err.code, err.desc); e != nil { - grpclog.Printf("grpc: Server.processUnaryRPC failed to write status %v", e) - } - return err - default: - if e := t.WriteStatus(stream, codes.Internal, err.Error()); e != nil { - grpclog.Printf("grpc: Server.processUnaryRPC failed to write status %v", e) - } - // TODO checkRecvPayload always return RPC error. Add a return here if necessary. - } + return err + } + if channelz.IsOn() { + t.IncrMsgRecv() + } + if st := checkRecvPayload(pf, stream.RecvCompress(), dc != nil || decomp != nil); st != nil { + if e := t.WriteStatus(stream, st); e != nil { + grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status %v", e) } - var inPayload *stats.InPayload - if stats.On() { - inPayload = &stats.InPayload{ - RecvTime: time.Now(), - } + return st.Err() + } + var inPayload *stats.InPayload + if sh != nil { + inPayload = &stats.InPayload{ + RecvTime: time.Now(), } - statusCode := codes.OK - statusDesc := "" - df := func(v interface{}) error { - if inPayload != nil { - inPayload.WireLength = len(req) - } - if pf == compressionMade { - var err error - req, err = s.opts.dc.Do(bytes.NewReader(req)) + } + df := func(v interface{}) error { + if inPayload != nil { + inPayload.WireLength = len(req) + } + if pf == compressionMade { + var err error + if dc != nil { + req, err = dc.Do(bytes.NewReader(req)) if err != nil { - if err := t.WriteStatus(stream, codes.Internal, err.Error()); err != nil { - grpclog.Printf("grpc: Server.processUnaryRPC failed to write status %v", err) - } - return Errorf(codes.Internal, err.Error()) + return status.Errorf(codes.Internal, err.Error()) } - } - if len(req) > s.opts.maxMsgSize { - // TODO: Revisit the error code. Currently keep it consistent with - // java implementation. - statusCode = codes.Internal - statusDesc = fmt.Sprintf("grpc: server received a message of %d bytes exceeding %d limit", len(req), s.opts.maxMsgSize) - } - if err := s.opts.codec.Unmarshal(req, v); err != nil { - return err - } - if inPayload != nil { - inPayload.Payload = v - inPayload.Data = req - inPayload.Length = len(req) - stats.HandleRPC(stream.Context(), inPayload) - } - if trInfo != nil { - trInfo.tr.LazyLog(&payload{sent: false, msg: v}, true) - } - return nil - } - reply, appErr := md.Handler(srv.server, stream.Context(), df, s.opts.unaryInt) - if appErr != nil { - if err, ok := appErr.(*rpcError); ok { - statusCode = err.code - statusDesc = err.desc } else { - statusCode = convertCode(appErr) - statusDesc = appErr.Error() - } - if trInfo != nil && statusCode != codes.OK { - trInfo.tr.LazyLog(stringer(statusDesc), true) - trInfo.tr.SetError() - } - if err := t.WriteStatus(stream, statusCode, statusDesc); err != nil { - grpclog.Printf("grpc: Server.processUnaryRPC failed to write status: %v", err) + tmp, _ := decomp.Decompress(bytes.NewReader(req)) + req, err = ioutil.ReadAll(tmp) + if err != nil { + return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + } } - return Errorf(statusCode, statusDesc) + } + if len(req) > s.opts.maxReceiveMessageSize { + // TODO: Revisit the error code. Currently keep it consistent with + // java implementation. + return status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", len(req), s.opts.maxReceiveMessageSize) + } + if err := s.getCodec(stream.ContentSubtype()).Unmarshal(req, v); err != nil { + return status.Errorf(codes.Internal, "grpc: error unmarshalling request: %v", err) + } + if inPayload != nil { + inPayload.Payload = v + inPayload.Data = req + inPayload.Length = len(req) + sh.HandleRPC(stream.Context(), inPayload) + } + if trInfo != nil { + trInfo.tr.LazyLog(&payload{sent: false, msg: v}, true) + } + return nil + } + ctx := NewContextWithServerTransportStream(stream.Context(), stream) + reply, appErr := md.Handler(srv.server, ctx, df, s.opts.unaryInt) + if appErr != nil { + appStatus, ok := status.FromError(appErr) + if !ok { + // Convert appErr if it is not a grpc status error. + appErr = status.Error(codes.Unknown, appErr.Error()) + appStatus, _ = status.FromError(appErr) } if trInfo != nil { - trInfo.tr.LazyLog(stringer("OK"), false) + trInfo.tr.LazyLog(stringer(appStatus.Message()), true) + trInfo.tr.SetError() } - opts := &transport.Options{ - Last: true, - Delay: false, + if e := t.WriteStatus(stream, appStatus); e != nil { + grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status: %v", e) } - if err := s.sendResponse(t, stream, reply, s.opts.cp, opts); err != nil { - switch err := err.(type) { + return appErr + } + if trInfo != nil { + trInfo.tr.LazyLog(stringer("OK"), false) + } + opts := &transport.Options{ + Last: true, + Delay: false, + } + + if err := s.sendResponse(t, stream, reply, cp, opts, comp); err != nil { + if err == io.EOF { + // The entire stream is done (for unary RPC only). + return err + } + if s, ok := status.FromError(err); ok { + if e := t.WriteStatus(stream, s); e != nil { + grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status: %v", e) + } + } else { + switch st := err.(type) { case transport.ConnectionError: // Nothing to do here. case transport.StreamError: - statusCode = err.Code - statusDesc = err.Desc + if e := t.WriteStatus(stream, status.New(st.Code, st.Desc)); e != nil { + grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status %v", e) + } default: - statusCode = codes.Unknown - statusDesc = err.Error() + panic(fmt.Sprintf("grpc: Unexpected error (%T) from sendResponse: %v", st, st)) } - return err } - if trInfo != nil { - trInfo.tr.LazyLog(&payload{sent: true, msg: reply}, true) - } - errWrite := t.WriteStatus(stream, statusCode, statusDesc) - if statusCode != codes.OK { - return Errorf(statusCode, statusDesc) - } - return errWrite + return err + } + if channelz.IsOn() { + t.IncrMsgSent() + } + if trInfo != nil { + trInfo.tr.LazyLog(&payload{sent: true, msg: reply}, true) } + // TODO: Should we be logging if writing status failed here, like above? + // Should the logging be in WriteStatus? Should we ignore the WriteStatus + // error or allow the stats handler to see it? + return t.WriteStatus(stream, status.New(codes.OK, "")) } func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, sd *StreamDesc, trInfo *traceInfo) (err error) { - if stats.On() { + if channelz.IsOn() { + s.incrCallsStarted() + defer func() { + if err != nil && err != io.EOF { + s.incrCallsFailed() + } else { + s.incrCallsSucceeded() + } + }() + } + sh := s.opts.statsHandler + if sh != nil { + beginTime := time.Now() begin := &stats.Begin{ - BeginTime: time.Now(), + BeginTime: beginTime, } - stats.HandleRPC(stream.Context(), begin) - } - defer func() { - if stats.On() { + sh.HandleRPC(stream.Context(), begin) + defer func() { end := &stats.End{ - EndTime: time.Now(), + BeginTime: beginTime, + EndTime: time.Now(), } if err != nil && err != io.EOF { end.Error = toRPCErr(err) } - stats.HandleRPC(stream.Context(), end) + sh.HandleRPC(stream.Context(), end) + }() + } + ctx := NewContextWithServerTransportStream(stream.Context(), stream) + ss := &serverStream{ + ctx: ctx, + t: t, + s: stream, + p: &parser{r: stream}, + codec: s.getCodec(stream.ContentSubtype()), + maxReceiveMessageSize: s.opts.maxReceiveMessageSize, + maxSendMessageSize: s.opts.maxSendMessageSize, + trInfo: trInfo, + statsHandler: sh, + } + + // If dc is set and matches the stream's compression, use it. Otherwise, try + // to find a matching registered compressor for decomp. + if rc := stream.RecvCompress(); s.opts.dc != nil && s.opts.dc.Type() == rc { + ss.dc = s.opts.dc + } else if rc != "" && rc != encoding.Identity { + ss.decomp = encoding.GetCompressor(rc) + if ss.decomp == nil { + st := status.Newf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", rc) + t.WriteStatus(ss.s, st) + return st.Err() } - }() + } + + // If cp is set, use it. Otherwise, attempt to compress the response using + // the incoming message compression method. + // + // NOTE: this needs to be ahead of all handling, https://github.com/grpc/grpc-go/issues/686. if s.opts.cp != nil { + ss.cp = s.opts.cp stream.SetSendCompress(s.opts.cp.Type()) + } else if rc := stream.RecvCompress(); rc != "" && rc != encoding.Identity { + // Legacy compressor not specified; attempt to respond with same encoding. + ss.comp = encoding.GetCompressor(rc) + if ss.comp != nil { + stream.SetSendCompress(rc) + } } - ss := &serverStream{ - t: t, - s: stream, - p: &parser{r: stream}, - codec: s.opts.codec, - cp: s.opts.cp, - dc: s.opts.dc, - maxMsgSize: s.opts.maxMsgSize, - trInfo: trInfo, - } - if ss.cp != nil { - ss.cbuf = new(bytes.Buffer) - } + if trInfo != nil { trInfo.tr.LazyLog(&trInfo.firstLine, false) defer func() { @@ -802,44 +1152,47 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp }() } var appErr error + var server interface{} + if srv != nil { + server = srv.server + } if s.opts.streamInt == nil { - appErr = sd.Handler(srv.server, ss) + appErr = sd.Handler(server, ss) } else { info := &StreamServerInfo{ FullMethod: stream.Method(), IsClientStream: sd.ClientStreams, IsServerStream: sd.ServerStreams, } - appErr = s.opts.streamInt(srv.server, ss, info, sd.Handler) + appErr = s.opts.streamInt(server, ss, info, sd.Handler) } if appErr != nil { - if err, ok := appErr.(*rpcError); ok { - ss.statusCode = err.code - ss.statusDesc = err.desc - } else if err, ok := appErr.(transport.StreamError); ok { - ss.statusCode = err.Code - ss.statusDesc = err.Desc - } else { - ss.statusCode = convertCode(appErr) - ss.statusDesc = appErr.Error() + appStatus, ok := status.FromError(appErr) + if !ok { + switch err := appErr.(type) { + case transport.StreamError: + appStatus = status.New(err.Code, err.Desc) + default: + appStatus = status.New(codes.Unknown, appErr.Error()) + } + appErr = appStatus.Err() } + if trInfo != nil { + ss.mu.Lock() + ss.trInfo.tr.LazyLog(stringer(appStatus.Message()), true) + ss.trInfo.tr.SetError() + ss.mu.Unlock() + } + t.WriteStatus(ss.s, appStatus) + // TODO: Should we log an error from WriteStatus here and below? + return appErr } if trInfo != nil { ss.mu.Lock() - if ss.statusCode != codes.OK { - ss.trInfo.tr.LazyLog(stringer(ss.statusDesc), true) - ss.trInfo.tr.SetError() - } else { - ss.trInfo.tr.LazyLog(stringer("OK"), false) - } + ss.trInfo.tr.LazyLog(stringer("OK"), false) ss.mu.Unlock() } - errWrite := t.WriteStatus(ss.s, ss.statusCode, ss.statusDesc) - if ss.statusCode != codes.OK { - return Errorf(ss.statusCode, ss.statusDesc) - } - return errWrite - + return t.WriteStatus(ss.s, status.New(codes.OK, "")) } func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Stream, trInfo *traceInfo) { @@ -854,12 +1207,12 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str trInfo.tr.SetError() } errDesc := fmt.Sprintf("malformed method name: %q", stream.Method()) - if err := t.WriteStatus(stream, codes.InvalidArgument, errDesc); err != nil { + if err := t.WriteStatus(stream, status.New(codes.ResourceExhausted, errDesc)); err != nil { if trInfo != nil { trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) trInfo.tr.SetError() } - grpclog.Printf("grpc: Server.handleStream failed to write status: %v", err) + grpclog.Warningf("grpc: Server.handleStream failed to write status: %v", err) } if trInfo != nil { trInfo.tr.Finish() @@ -870,17 +1223,21 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str method := sm[pos+1:] srv, ok := s.m[service] if !ok { + if unknownDesc := s.opts.unknownStreamDesc; unknownDesc != nil { + s.processStreamingRPC(t, stream, nil, unknownDesc, trInfo) + return + } if trInfo != nil { trInfo.tr.LazyLog(&fmtStringer{"Unknown service %v", []interface{}{service}}, true) trInfo.tr.SetError() } errDesc := fmt.Sprintf("unknown service %v", service) - if err := t.WriteStatus(stream, codes.Unimplemented, errDesc); err != nil { + if err := t.WriteStatus(stream, status.New(codes.Unimplemented, errDesc)); err != nil { if trInfo != nil { trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) trInfo.tr.SetError() } - grpclog.Printf("grpc: Server.handleStream failed to write status: %v", err) + grpclog.Warningf("grpc: Server.handleStream failed to write status: %v", err) } if trInfo != nil { trInfo.tr.Finish() @@ -900,25 +1257,82 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str trInfo.tr.LazyLog(&fmtStringer{"Unknown method %v", []interface{}{method}}, true) trInfo.tr.SetError() } + if unknownDesc := s.opts.unknownStreamDesc; unknownDesc != nil { + s.processStreamingRPC(t, stream, nil, unknownDesc, trInfo) + return + } errDesc := fmt.Sprintf("unknown method %v", method) - if err := t.WriteStatus(stream, codes.Unimplemented, errDesc); err != nil { + if err := t.WriteStatus(stream, status.New(codes.Unimplemented, errDesc)); err != nil { if trInfo != nil { trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) trInfo.tr.SetError() } - grpclog.Printf("grpc: Server.handleStream failed to write status: %v", err) + grpclog.Warningf("grpc: Server.handleStream failed to write status: %v", err) } if trInfo != nil { trInfo.tr.Finish() } } +// The key to save ServerTransportStream in the context. +type streamKey struct{} + +// NewContextWithServerTransportStream creates a new context from ctx and +// attaches stream to it. +// +// This API is EXPERIMENTAL. +func NewContextWithServerTransportStream(ctx context.Context, stream ServerTransportStream) context.Context { + return context.WithValue(ctx, streamKey{}, stream) +} + +// ServerTransportStream is a minimal interface that a transport stream must +// implement. This can be used to mock an actual transport stream for tests of +// handler code that use, for example, grpc.SetHeader (which requires some +// stream to be in context). +// +// See also NewContextWithServerTransportStream. +// +// This API is EXPERIMENTAL. +type ServerTransportStream interface { + Method() string + SetHeader(md metadata.MD) error + SendHeader(md metadata.MD) error + SetTrailer(md metadata.MD) error +} + +// ServerTransportStreamFromContext returns the ServerTransportStream saved in +// ctx. Returns nil if the given context has no stream associated with it +// (which implies it is not an RPC invocation context). +// +// This API is EXPERIMENTAL. +func ServerTransportStreamFromContext(ctx context.Context) ServerTransportStream { + s, _ := ctx.Value(streamKey{}).(ServerTransportStream) + return s +} + // Stop stops the gRPC server. It immediately closes all open // connections and listeners. // It cancels all active RPCs on the server side and the corresponding // pending RPCs on the client side will get notified by connection // errors. func (s *Server) Stop() { + s.quitOnce.Do(func() { + close(s.quit) + }) + + defer func() { + s.serveWG.Wait() + s.doneOnce.Do(func() { + close(s.done) + }) + }() + + s.channelzRemoveOnce.Do(func() { + if channelz.IsOn() { + channelz.RemoveEntry(s.channelzID) + } + }) + s.mu.Lock() listeners := s.lis s.lis = nil @@ -936,7 +1350,6 @@ func (s *Server) Stop() { } s.mu.Lock() - s.cancel() if s.events != nil { s.events.Finish() s.events = nil @@ -944,25 +1357,48 @@ func (s *Server) Stop() { s.mu.Unlock() } -// GracefulStop stops the gRPC server gracefully. It stops the server to accept new -// connections and RPCs and blocks until all the pending RPCs are finished. +// GracefulStop stops the gRPC server gracefully. It stops the server from +// accepting new connections and RPCs and blocks until all the pending RPCs are +// finished. func (s *Server) GracefulStop() { + s.quitOnce.Do(func() { + close(s.quit) + }) + + defer func() { + s.doneOnce.Do(func() { + close(s.done) + }) + }() + + s.channelzRemoveOnce.Do(func() { + if channelz.IsOn() { + channelz.RemoveEntry(s.channelzID) + } + }) s.mu.Lock() - defer s.mu.Unlock() if s.conns == nil { + s.mu.Unlock() return } + for lis := range s.lis { lis.Close() } s.lis = nil - s.cancel() if !s.drain { for c := range s.conns { c.(transport.ServerTransport).Drain() } s.drain = true } + + // Wait for serving threads to be ready to exit. Only then can we be sure no + // new conns will be created. + s.mu.Unlock() + s.serveWG.Wait() + s.mu.Lock() + for len(s.conns) != 0 { s.cv.Wait() } @@ -971,26 +1407,29 @@ func (s *Server) GracefulStop() { s.events.Finish() s.events = nil } + s.mu.Unlock() } func init() { - internal.TestingCloseConns = func(arg interface{}) { - arg.(*Server).testingCloseConns() - } internal.TestingUseHandlerImpl = func(arg interface{}) { arg.(*Server).opts.useHandlerImpl = true } } -// testingCloseConns closes all existing transports but keeps s.lis -// accepting new connections. -func (s *Server) testingCloseConns() { - s.mu.Lock() - for c := range s.conns { - c.Close() - delete(s.conns, c) +// contentSubtype must be lowercase +// cannot return nil +func (s *Server) getCodec(contentSubtype string) baseCodec { + if s.opts.codec != nil { + return s.opts.codec } - s.mu.Unlock() + if contentSubtype == "" { + return encoding.GetCodec(proto.Name) + } + codec := encoding.GetCodec(contentSubtype) + if codec == nil { + return encoding.GetCodec(proto.Name) + } + return codec } // SetHeader sets the header metadata. @@ -1003,9 +1442,9 @@ func SetHeader(ctx context.Context, md metadata.MD) error { if md.Len() == 0 { return nil } - stream, ok := transport.StreamFromContext(ctx) - if !ok { - return Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) + stream := ServerTransportStreamFromContext(ctx) + if stream == nil { + return status.Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) } return stream.SetHeader(md) } @@ -1013,15 +1452,11 @@ func SetHeader(ctx context.Context, md metadata.MD) error { // SendHeader sends header metadata. It may be called at most once. // The provided md and headers set by SetHeader() will be sent. func SendHeader(ctx context.Context, md metadata.MD) error { - stream, ok := transport.StreamFromContext(ctx) - if !ok { - return Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) + stream := ServerTransportStreamFromContext(ctx) + if stream == nil { + return status.Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) } - t := stream.ServerTransport() - if t == nil { - grpclog.Fatalf("grpc: SendHeader: %v has no ServerTransport to send header metadata.", stream) - } - if err := t.WriteHeader(stream, md); err != nil { + if err := stream.SendHeader(md); err != nil { return toRPCErr(err) } return nil @@ -1033,9 +1468,19 @@ func SetTrailer(ctx context.Context, md metadata.MD) error { if md.Len() == 0 { return nil } - stream, ok := transport.StreamFromContext(ctx) - if !ok { - return Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) + stream := ServerTransportStreamFromContext(ctx) + if stream == nil { + return status.Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) } return stream.SetTrailer(md) } + +// Method returns the method string for the server context. The returned +// string is in the format of "/service/method". +func Method(ctx context.Context) (string, bool) { + s := ServerTransportStreamFromContext(ctx) + if s == nil { + return "", false + } + return s.Method(), true +} diff --git a/vendor/google.golang.org/grpc/service_config.go b/vendor/google.golang.org/grpc/service_config.go new file mode 100644 index 00000000000..015631d8d38 --- /dev/null +++ b/vendor/google.golang.org/grpc/service_config.go @@ -0,0 +1,233 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "google.golang.org/grpc/grpclog" +) + +const maxInt = int(^uint(0) >> 1) + +// MethodConfig defines the configuration recommended by the service providers for a +// particular method. +// +// Deprecated: Users should not use this struct. Service config should be received +// through name resolver, as specified here +// https://github.com/grpc/grpc/blob/master/doc/service_config.md +type MethodConfig struct { + // WaitForReady indicates whether RPCs sent to this method should wait until + // the connection is ready by default (!failfast). The value specified via the + // gRPC client API will override the value set here. + WaitForReady *bool + // Timeout is the default timeout for RPCs sent to this method. The actual + // deadline used will be the minimum of the value specified here and the value + // set by the application via the gRPC client API. If either one is not set, + // then the other will be used. If neither is set, then the RPC has no deadline. + Timeout *time.Duration + // MaxReqSize is the maximum allowed payload size for an individual request in a + // stream (client->server) in bytes. The size which is measured is the serialized + // payload after per-message compression (but before stream compression) in bytes. + // The actual value used is the minimum of the value specified here and the value set + // by the application via the gRPC client API. If either one is not set, then the other + // will be used. If neither is set, then the built-in default is used. + MaxReqSize *int + // MaxRespSize is the maximum allowed payload size for an individual response in a + // stream (server->client) in bytes. + MaxRespSize *int +} + +// ServiceConfig is provided by the service provider and contains parameters for how +// clients that connect to the service should behave. +// +// Deprecated: Users should not use this struct. Service config should be received +// through name resolver, as specified here +// https://github.com/grpc/grpc/blob/master/doc/service_config.md +type ServiceConfig struct { + // LB is the load balancer the service providers recommends. The balancer specified + // via grpc.WithBalancer will override this. + LB *string + // Methods contains a map for the methods in this service. + // If there is an exact match for a method (i.e. /service/method) in the map, use the corresponding MethodConfig. + // If there's no exact match, look for the default config for the service (/service/) and use the corresponding MethodConfig if it exists. + // Otherwise, the method has no MethodConfig to use. + Methods map[string]MethodConfig + + stickinessMetadataKey *string +} + +func parseDuration(s *string) (*time.Duration, error) { + if s == nil { + return nil, nil + } + if !strings.HasSuffix(*s, "s") { + return nil, fmt.Errorf("malformed duration %q", *s) + } + ss := strings.SplitN((*s)[:len(*s)-1], ".", 3) + if len(ss) > 2 { + return nil, fmt.Errorf("malformed duration %q", *s) + } + // hasDigits is set if either the whole or fractional part of the number is + // present, since both are optional but one is required. + hasDigits := false + var d time.Duration + if len(ss[0]) > 0 { + i, err := strconv.ParseInt(ss[0], 10, 32) + if err != nil { + return nil, fmt.Errorf("malformed duration %q: %v", *s, err) + } + d = time.Duration(i) * time.Second + hasDigits = true + } + if len(ss) == 2 && len(ss[1]) > 0 { + if len(ss[1]) > 9 { + return nil, fmt.Errorf("malformed duration %q", *s) + } + f, err := strconv.ParseInt(ss[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("malformed duration %q: %v", *s, err) + } + for i := 9; i > len(ss[1]); i-- { + f *= 10 + } + d += time.Duration(f) + hasDigits = true + } + if !hasDigits { + return nil, fmt.Errorf("malformed duration %q", *s) + } + + return &d, nil +} + +type jsonName struct { + Service *string + Method *string +} + +func (j jsonName) generatePath() (string, bool) { + if j.Service == nil { + return "", false + } + res := "/" + *j.Service + "/" + if j.Method != nil { + res += *j.Method + } + return res, true +} + +// TODO(lyuxuan): delete this struct after cleaning up old service config implementation. +type jsonMC struct { + Name *[]jsonName + WaitForReady *bool + Timeout *string + MaxRequestMessageBytes *int64 + MaxResponseMessageBytes *int64 +} + +// TODO(lyuxuan): delete this struct after cleaning up old service config implementation. +type jsonSC struct { + LoadBalancingPolicy *string + StickinessMetadataKey *string + MethodConfig *[]jsonMC +} + +func parseServiceConfig(js string) (ServiceConfig, error) { + var rsc jsonSC + err := json.Unmarshal([]byte(js), &rsc) + if err != nil { + grpclog.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err) + return ServiceConfig{}, err + } + sc := ServiceConfig{ + LB: rsc.LoadBalancingPolicy, + Methods: make(map[string]MethodConfig), + + stickinessMetadataKey: rsc.StickinessMetadataKey, + } + if rsc.MethodConfig == nil { + return sc, nil + } + + for _, m := range *rsc.MethodConfig { + if m.Name == nil { + continue + } + d, err := parseDuration(m.Timeout) + if err != nil { + grpclog.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err) + return ServiceConfig{}, err + } + + mc := MethodConfig{ + WaitForReady: m.WaitForReady, + Timeout: d, + } + if m.MaxRequestMessageBytes != nil { + if *m.MaxRequestMessageBytes > int64(maxInt) { + mc.MaxReqSize = newInt(maxInt) + } else { + mc.MaxReqSize = newInt(int(*m.MaxRequestMessageBytes)) + } + } + if m.MaxResponseMessageBytes != nil { + if *m.MaxResponseMessageBytes > int64(maxInt) { + mc.MaxRespSize = newInt(maxInt) + } else { + mc.MaxRespSize = newInt(int(*m.MaxResponseMessageBytes)) + } + } + for _, n := range *m.Name { + if path, valid := n.generatePath(); valid { + sc.Methods[path] = mc + } + } + } + + return sc, nil +} + +func min(a, b *int) *int { + if *a < *b { + return a + } + return b +} + +func getMaxSize(mcMax, doptMax *int, defaultVal int) *int { + if mcMax == nil && doptMax == nil { + return &defaultVal + } + if mcMax != nil && doptMax != nil { + return min(mcMax, doptMax) + } + if mcMax != nil { + return mcMax + } + return doptMax +} + +func newInt(b int) *int { + return &b +} diff --git a/vendor/google.golang.org/grpc/stats/handlers.go b/vendor/google.golang.org/grpc/stats/handlers.go index d41c524421d..05b384c6931 100644 --- a/vendor/google.golang.org/grpc/stats/handlers.go +++ b/vendor/google.golang.org/grpc/stats/handlers.go @@ -1,33 +1,18 @@ /* * - * Copyright 2016, Google Inc. - * All rights reserved. + * Copyright 2016 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -35,10 +20,8 @@ package stats import ( "net" - "sync/atomic" "golang.org/x/net/context" - "google.golang.org/grpc/grpclog" ) // ConnTagInfo defines the relevant information needed by connection context tagger. @@ -47,106 +30,35 @@ type ConnTagInfo struct { RemoteAddr net.Addr // LocalAddr is the local address of the corresponding connection. LocalAddr net.Addr - // TODO add QOS related fields. } // RPCTagInfo defines the relevant information needed by RPC context tagger. type RPCTagInfo struct { // FullMethodName is the RPC method in the format of /package.service/method. FullMethodName string + // FailFast indicates if this RPC is failfast. + // This field is only valid on client side, it's always false on server side. + FailFast bool } -var ( - on = new(int32) - rpcHandler func(context.Context, RPCStats) - connHandler func(context.Context, ConnStats) - connTagger func(context.Context, *ConnTagInfo) context.Context - rpcTagger func(context.Context, *RPCTagInfo) context.Context -) - -// HandleRPC processes the RPC stats using the rpc handler registered by the user. -func HandleRPC(ctx context.Context, s RPCStats) { - if rpcHandler == nil { - return - } - rpcHandler(ctx, s) -} - -// RegisterRPCHandler registers the user handler function for RPC stats processing. -// It should be called only once. The later call will overwrite the former value if it is called multiple times. -// This handler function will be called to process the rpc stats. -func RegisterRPCHandler(f func(context.Context, RPCStats)) { - rpcHandler = f -} - -// HandleConn processes the stats using the call back function registered by user. -func HandleConn(ctx context.Context, s ConnStats) { - if connHandler == nil { - return - } - connHandler(ctx, s) -} - -// RegisterConnHandler registers the user handler function for conn stats. -// It should be called only once. The later call will overwrite the former value if it is called multiple times. -// This handler function will be called to process the conn stats. -func RegisterConnHandler(f func(context.Context, ConnStats)) { - connHandler = f -} - -// TagConn calls user registered connection context tagger. -func TagConn(ctx context.Context, info *ConnTagInfo) context.Context { - if connTagger == nil { - return ctx - } - return connTagger(ctx, info) -} - -// RegisterConnTagger registers the user connection context tagger function. -// The connection context tagger can attach some information to the given context. -// The returned context will be used for stats handling. -// For conn stats handling, the context used in connHandler for this -// connection will be derived from the context returned. -// For RPC stats handling, -// - On server side, the context used in rpcHandler for all RPCs on this -// connection will be derived from the context returned. -// - On client side, the context is not derived from the context returned. -func RegisterConnTagger(t func(context.Context, *ConnTagInfo) context.Context) { - connTagger = t -} - -// TagRPC calls the user registered RPC context tagger. -func TagRPC(ctx context.Context, info *RPCTagInfo) context.Context { - if rpcTagger == nil { - return ctx - } - return rpcTagger(ctx, info) -} - -// RegisterRPCTagger registers the user RPC context tagger function. -// The RPC context tagger can attach some information to the given context. -// The context used in stats rpcHandler for this RPC will be derived from the -// context returned. -func RegisterRPCTagger(t func(context.Context, *RPCTagInfo) context.Context) { - rpcTagger = t -} - -// Start starts the stats collection and processing if there is a registered stats handle. -func Start() { - if rpcHandler == nil && connHandler == nil { - grpclog.Println("rpcHandler and connHandler are both nil when starting stats. Stats is not started") - return - } - atomic.StoreInt32(on, 1) -} - -// Stop stops the stats collection and processing. -// Stop does not unregister the handlers. -func Stop() { - atomic.StoreInt32(on, 0) -} - -// On indicates whether the stats collection and processing is on. -func On() bool { - return atomic.CompareAndSwapInt32(on, 1, 1) +// Handler defines the interface for the related stats handling (e.g., RPCs, connections). +type Handler interface { + // TagRPC can attach some information to the given context. + // The context used for the rest lifetime of the RPC will be derived from + // the returned context. + TagRPC(context.Context, *RPCTagInfo) context.Context + // HandleRPC processes the RPC stats. + HandleRPC(context.Context, RPCStats) + + // TagConn can attach some information to the given context. + // The returned context will be used for stats handling. + // For conn stats handling, the context used in HandleConn for this + // connection will be derived from the context returned. + // For RPC stats handling, + // - On server side, the context used in HandleRPC for all RPCs on this + // connection will be derived from the context returned. + // - On client side, the context is not derived from the context returned. + TagConn(context.Context, *ConnTagInfo) context.Context + // HandleConn processes the Conn stats. + HandleConn(context.Context, ConnStats) } diff --git a/vendor/google.golang.org/grpc/stats/stats.go b/vendor/google.golang.org/grpc/stats/stats.go index a82448a68bb..3f13190a0ac 100644 --- a/vendor/google.golang.org/grpc/stats/stats.go +++ b/vendor/google.golang.org/grpc/stats/stats.go @@ -1,36 +1,23 @@ /* * - * Copyright 2016, Google Inc. - * All rights reserved. + * Copyright 2016 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ +//go:generate protoc --go_out=plugins=grpc:. grpc_testing/test.proto + // Package stats is for collecting and reporting various network and RPC stats. // This package is for monitoring purpose only. All fields are read-only. // All APIs are experimental. @@ -39,6 +26,8 @@ package stats // import "google.golang.org/grpc/stats" import ( "net" "time" + + "golang.org/x/net/context" ) // RPCStats contains stats information about RPCs. @@ -49,7 +38,7 @@ type RPCStats interface { } // Begin contains stats when an RPC begins. -// FailFast are only valid if Client is true. +// FailFast is only valid if this Begin is from client side. type Begin struct { // Client is true if this Begin is from client side. Client bool @@ -59,7 +48,7 @@ type Begin struct { FailFast bool } -// IsClient indicates if this is from client side. +// IsClient indicates if the stats information is from client side. func (s *Begin) IsClient() bool { return s.Client } func (s *Begin) isRPCStats() {} @@ -80,19 +69,19 @@ type InPayload struct { RecvTime time.Time } -// IsClient indicates if this is from client side. +// IsClient indicates if the stats information is from client side. func (s *InPayload) IsClient() bool { return s.Client } func (s *InPayload) isRPCStats() {} // InHeader contains stats when a header is received. -// FullMethod, addresses and Compression are only valid if Client is false. type InHeader struct { // Client is true if this InHeader is from client side. Client bool // WireLength is the wire length of header. WireLength int + // The following fields are valid only if Client is false. // FullMethod is the full RPC method string, i.e., /package.service/method. FullMethod string // RemoteAddr is the remote address of the corresponding connection. @@ -103,7 +92,7 @@ type InHeader struct { Compression string } -// IsClient indicates if this is from client side. +// IsClient indicates if the stats information is from client side. func (s *InHeader) IsClient() bool { return s.Client } func (s *InHeader) isRPCStats() {} @@ -116,7 +105,7 @@ type InTrailer struct { WireLength int } -// IsClient indicates if this is from client side. +// IsClient indicates if the stats information is from client side. func (s *InTrailer) IsClient() bool { return s.Client } func (s *InTrailer) isRPCStats() {} @@ -137,19 +126,17 @@ type OutPayload struct { SentTime time.Time } -// IsClient indicates if this is from client side. +// IsClient indicates if this stats information is from client side. func (s *OutPayload) IsClient() bool { return s.Client } func (s *OutPayload) isRPCStats() {} // OutHeader contains stats when a header is sent. -// FullMethod, addresses and Compression are only valid if Client is true. type OutHeader struct { // Client is true if this OutHeader is from client side. Client bool - // WireLength is the wire length of header. - WireLength int + // The following fields are valid only if Client is true. // FullMethod is the full RPC method string, i.e., /package.service/method. FullMethod string // RemoteAddr is the remote address of the corresponding connection. @@ -160,7 +147,7 @@ type OutHeader struct { Compression string } -// IsClient indicates if this is from client side. +// IsClient indicates if this stats information is from client side. func (s *OutHeader) IsClient() bool { return s.Client } func (s *OutHeader) isRPCStats() {} @@ -173,7 +160,7 @@ type OutTrailer struct { WireLength int } -// IsClient indicates if this is from client side. +// IsClient indicates if this stats information is from client side. func (s *OutTrailer) IsClient() bool { return s.Client } func (s *OutTrailer) isRPCStats() {} @@ -182,9 +169,13 @@ func (s *OutTrailer) isRPCStats() {} type End struct { // Client is true if this End is from client side. Client bool + // BeginTime is the time when the RPC began. + BeginTime time.Time // EndTime is the time when the RPC ends. EndTime time.Time - // Error is the error just happened. Its type is gRPC error. + // Error is the error the RPC ended with. It is an error generated from + // status.Status and can be converted back to status.Status using + // status.FromError if non-nil. Error error } @@ -221,3 +212,85 @@ type ConnEnd struct { func (s *ConnEnd) IsClient() bool { return s.Client } func (s *ConnEnd) isConnStats() {} + +type incomingTagsKey struct{} +type outgoingTagsKey struct{} + +// SetTags attaches stats tagging data to the context, which will be sent in +// the outgoing RPC with the header grpc-tags-bin. Subsequent calls to +// SetTags will overwrite the values from earlier calls. +// +// NOTE: this is provided only for backward compatibility with existing clients +// and will likely be removed in an upcoming release. New uses should transmit +// this type of data using metadata with a different, non-reserved (i.e. does +// not begin with "grpc-") header name. +func SetTags(ctx context.Context, b []byte) context.Context { + return context.WithValue(ctx, outgoingTagsKey{}, b) +} + +// Tags returns the tags from the context for the inbound RPC. +// +// NOTE: this is provided only for backward compatibility with existing clients +// and will likely be removed in an upcoming release. New uses should transmit +// this type of data using metadata with a different, non-reserved (i.e. does +// not begin with "grpc-") header name. +func Tags(ctx context.Context) []byte { + b, _ := ctx.Value(incomingTagsKey{}).([]byte) + return b +} + +// SetIncomingTags attaches stats tagging data to the context, to be read by +// the application (not sent in outgoing RPCs). +// +// This is intended for gRPC-internal use ONLY. +func SetIncomingTags(ctx context.Context, b []byte) context.Context { + return context.WithValue(ctx, incomingTagsKey{}, b) +} + +// OutgoingTags returns the tags from the context for the outbound RPC. +// +// This is intended for gRPC-internal use ONLY. +func OutgoingTags(ctx context.Context) []byte { + b, _ := ctx.Value(outgoingTagsKey{}).([]byte) + return b +} + +type incomingTraceKey struct{} +type outgoingTraceKey struct{} + +// SetTrace attaches stats tagging data to the context, which will be sent in +// the outgoing RPC with the header grpc-trace-bin. Subsequent calls to +// SetTrace will overwrite the values from earlier calls. +// +// NOTE: this is provided only for backward compatibility with existing clients +// and will likely be removed in an upcoming release. New uses should transmit +// this type of data using metadata with a different, non-reserved (i.e. does +// not begin with "grpc-") header name. +func SetTrace(ctx context.Context, b []byte) context.Context { + return context.WithValue(ctx, outgoingTraceKey{}, b) +} + +// Trace returns the trace from the context for the inbound RPC. +// +// NOTE: this is provided only for backward compatibility with existing clients +// and will likely be removed in an upcoming release. New uses should transmit +// this type of data using metadata with a different, non-reserved (i.e. does +// not begin with "grpc-") header name. +func Trace(ctx context.Context) []byte { + b, _ := ctx.Value(incomingTraceKey{}).([]byte) + return b +} + +// SetIncomingTrace attaches stats tagging data to the context, to be read by +// the application (not sent in outgoing RPCs). It is intended for +// gRPC-internal use. +func SetIncomingTrace(ctx context.Context, b []byte) context.Context { + return context.WithValue(ctx, incomingTraceKey{}, b) +} + +// OutgoingTrace returns the trace from the context for the outbound RPC. It is +// intended for gRPC-internal use. +func OutgoingTrace(ctx context.Context) []byte { + b, _ := ctx.Value(outgoingTraceKey{}).([]byte) + return b +} diff --git a/vendor/google.golang.org/grpc/status/status.go b/vendor/google.golang.org/grpc/status/status.go new file mode 100644 index 00000000000..9c61b094508 --- /dev/null +++ b/vendor/google.golang.org/grpc/status/status.go @@ -0,0 +1,189 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package status implements errors returned by gRPC. These errors are +// serialized and transmitted on the wire between server and client, and allow +// for additional data to be transmitted via the Details field in the status +// proto. gRPC service handlers should return an error created by this +// package, and gRPC clients should expect a corresponding error to be +// returned from the RPC call. +// +// This package upholds the invariants that a non-nil error may not +// contain an OK code, and an OK code must result in a nil error. +package status + +import ( + "errors" + "fmt" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + spb "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc/codes" +) + +// statusError is an alias of a status proto. It implements error and Status, +// and a nil statusError should never be returned by this package. +type statusError spb.Status + +func (se *statusError) Error() string { + p := (*spb.Status)(se) + return fmt.Sprintf("rpc error: code = %s desc = %s", codes.Code(p.GetCode()), p.GetMessage()) +} + +func (se *statusError) GRPCStatus() *Status { + return &Status{s: (*spb.Status)(se)} +} + +// Status represents an RPC status code, message, and details. It is immutable +// and should be created with New, Newf, or FromProto. +type Status struct { + s *spb.Status +} + +// Code returns the status code contained in s. +func (s *Status) Code() codes.Code { + if s == nil || s.s == nil { + return codes.OK + } + return codes.Code(s.s.Code) +} + +// Message returns the message contained in s. +func (s *Status) Message() string { + if s == nil || s.s == nil { + return "" + } + return s.s.Message +} + +// Proto returns s's status as an spb.Status proto message. +func (s *Status) Proto() *spb.Status { + if s == nil { + return nil + } + return proto.Clone(s.s).(*spb.Status) +} + +// Err returns an immutable error representing s; returns nil if s.Code() is +// OK. +func (s *Status) Err() error { + if s.Code() == codes.OK { + return nil + } + return (*statusError)(s.s) +} + +// New returns a Status representing c and msg. +func New(c codes.Code, msg string) *Status { + return &Status{s: &spb.Status{Code: int32(c), Message: msg}} +} + +// Newf returns New(c, fmt.Sprintf(format, a...)). +func Newf(c codes.Code, format string, a ...interface{}) *Status { + return New(c, fmt.Sprintf(format, a...)) +} + +// Error returns an error representing c and msg. If c is OK, returns nil. +func Error(c codes.Code, msg string) error { + return New(c, msg).Err() +} + +// Errorf returns Error(c, fmt.Sprintf(format, a...)). +func Errorf(c codes.Code, format string, a ...interface{}) error { + return Error(c, fmt.Sprintf(format, a...)) +} + +// ErrorProto returns an error representing s. If s.Code is OK, returns nil. +func ErrorProto(s *spb.Status) error { + return FromProto(s).Err() +} + +// FromProto returns a Status representing s. +func FromProto(s *spb.Status) *Status { + return &Status{s: proto.Clone(s).(*spb.Status)} +} + +// FromError returns a Status representing err if it was produced from this +// package or has a method `GRPCStatus() *Status`. Otherwise, ok is false and a +// Status is returned with codes.Unknown and the original error message. +func FromError(err error) (s *Status, ok bool) { + if err == nil { + return &Status{s: &spb.Status{Code: int32(codes.OK)}}, true + } + if se, ok := err.(interface{ GRPCStatus() *Status }); ok { + return se.GRPCStatus(), true + } + return New(codes.Unknown, err.Error()), false +} + +// Convert is a convenience function which removes the need to handle the +// boolean return value from FromError. +func Convert(err error) *Status { + s, _ := FromError(err) + return s +} + +// WithDetails returns a new status with the provided details messages appended to the status. +// If any errors are encountered, it returns nil and the first error encountered. +func (s *Status) WithDetails(details ...proto.Message) (*Status, error) { + if s.Code() == codes.OK { + return nil, errors.New("no error details for status with code OK") + } + // s.Code() != OK implies that s.Proto() != nil. + p := s.Proto() + for _, detail := range details { + any, err := ptypes.MarshalAny(detail) + if err != nil { + return nil, err + } + p.Details = append(p.Details, any) + } + return &Status{s: p}, nil +} + +// Details returns a slice of details messages attached to the status. +// If a detail cannot be decoded, the error is returned in place of the detail. +func (s *Status) Details() []interface{} { + if s == nil || s.s == nil { + return nil + } + details := make([]interface{}, 0, len(s.s.Details)) + for _, any := range s.s.Details { + detail := &ptypes.DynamicAny{} + if err := ptypes.UnmarshalAny(any, detail); err != nil { + details = append(details, err) + continue + } + details = append(details, detail.Message) + } + return details +} + +// Code returns the Code of the error if it is a Status error, codes.OK if err +// is nil, or codes.Unknown otherwise. +func Code(err error) codes.Code { + // Don't use FromError to avoid allocation of OK status. + if err == nil { + return codes.OK + } + if se, ok := err.(interface{ GRPCStatus() *Status }); ok { + return se.GRPCStatus().Code() + } + return codes.Unknown +} diff --git a/vendor/google.golang.org/grpc/stream.go b/vendor/google.golang.org/grpc/stream.go index 1bcd2183a8c..82921a15a3a 100644 --- a/vendor/google.golang.org/grpc/stream.go +++ b/vendor/google.golang.org/grpc/stream.go @@ -1,56 +1,46 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ package grpc import ( - "bytes" "errors" "io" - "math" "sync" "time" "golang.org/x/net/context" "golang.org/x/net/trace" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/channelz" "google.golang.org/grpc/codes" + "google.golang.org/grpc/encoding" "google.golang.org/grpc/metadata" "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" "google.golang.org/grpc/transport" ) // StreamHandler defines the handler called by gRPC server to complete the -// execution of a streaming RPC. +// execution of a streaming RPC. If a StreamHandler returns an error, it +// should be produced by the status package, or else gRPC will use +// codes.Unknown as the status code and err.Error() as the status message +// of the RPC. type StreamHandler func(srv interface{}, stream ServerStream) error // StreamDesc represents a streaming RPC service's method specification. @@ -64,6 +54,8 @@ type StreamDesc struct { } // Stream defines the common interface a client or server stream has to satisfy. +// +// All errors returned from Stream are compatible with the status package. type Stream interface { // Context returns the context for this stream. Context() context.Context @@ -73,11 +65,17 @@ type Stream interface { // side. On server side, it simply returns the error to the caller. // SendMsg is called by generated code. Also Users can call SendMsg // directly when it is really needed in their use cases. + // It's safe to have a goroutine calling SendMsg and another goroutine calling + // recvMsg on the same stream at the same time. + // But it is not safe to call SendMsg on the same stream in different goroutines. SendMsg(m interface{}) error // RecvMsg blocks until it receives a message or the stream is // done. On client side, it returns io.EOF when the stream is done. On // any other error, it aborts the stream and returns an RPC status. On // server side, it simply returns the error to the caller. + // It's safe to have a goroutine calling SendMsg and another goroutine calling + // recvMsg on the same stream at the same time. + // But it is not safe to call RecvMsg on the same stream in different goroutines. RecvMsg(m interface{}) error } @@ -93,37 +91,110 @@ type ClientStream interface { // CloseSend closes the send direction of the stream. It closes the stream // when non-nil error is met. CloseSend() error + // Stream.SendMsg() may return a non-nil error when something wrong happens sending + // the request. The returned error indicates the status of this sending, not the final + // status of the RPC. + // + // Always call Stream.RecvMsg() to drain the stream and get the final + // status, otherwise there could be leaked resources. Stream } -// NewClientStream creates a new Stream for the client side. This is called -// by generated code. -func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) { +// NewStream creates a new Stream for the client side. This is typically +// called by generated code. +func (cc *ClientConn) NewStream(ctx context.Context, desc *StreamDesc, method string, opts ...CallOption) (ClientStream, error) { + // allow interceptor to see all applicable call options, which means those + // configured as defaults from dial option as well as per-call options + opts = combine(cc.dopts.callOptions, opts) + if cc.dopts.streamInt != nil { return cc.dopts.streamInt(ctx, desc, cc, method, newClientStream, opts...) } return newClientStream(ctx, desc, cc, method, opts...) } +// NewClientStream creates a new Stream for the client side. This is typically +// called by generated code. +// +// DEPRECATED: Use ClientConn.NewStream instead. +func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) { + return cc.NewStream(ctx, desc, method, opts...) +} + func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) { - var ( - t transport.ClientTransport - s *transport.Stream - put func() - ) - c := defaultCallInfo + if channelz.IsOn() { + cc.incrCallsStarted() + defer func() { + if err != nil { + cc.incrCallsFailed() + } + }() + } + c := defaultCallInfo() + mc := cc.GetMethodConfig(method) + if mc.WaitForReady != nil { + c.failFast = !*mc.WaitForReady + } + + // Possible context leak: + // The cancel function for the child context we create will only be called + // when RecvMsg returns a non-nil error, if the ClientConn is closed, or if + // an error is generated by SendMsg. + // https://github.com/grpc/grpc-go/issues/1818. + var cancel context.CancelFunc + if mc.Timeout != nil && *mc.Timeout >= 0 { + ctx, cancel = context.WithTimeout(ctx, *mc.Timeout) + } else { + ctx, cancel = context.WithCancel(ctx) + } + defer func() { + if err != nil { + cancel() + } + }() + for _, o := range opts { - if err := o.before(&c); err != nil { + if err := o.before(c); err != nil { return nil, toRPCErr(err) } } + c.maxSendMessageSize = getMaxSize(mc.MaxReqSize, c.maxSendMessageSize, defaultClientMaxSendMessageSize) + c.maxReceiveMessageSize = getMaxSize(mc.MaxRespSize, c.maxReceiveMessageSize, defaultClientMaxReceiveMessageSize) + if err := setCallInfoCodec(c); err != nil { + return nil, err + } + callHdr := &transport.CallHdr{ Host: cc.authority, Method: method, - Flush: desc.ServerStreams && desc.ClientStreams, - } - if cc.dopts.cp != nil { + // If it's not client streaming, we should already have the request to be sent, + // so we don't flush the header. + // If it's client streaming, the user may never send a request or send it any + // time soon, so we ask the transport to flush the header. + Flush: desc.ClientStreams, + ContentSubtype: c.contentSubtype, + } + + // Set our outgoing compression according to the UseCompressor CallOption, if + // set. In that case, also find the compressor from the encoding package. + // Otherwise, use the compressor configured by the WithCompressor DialOption, + // if set. + var cp Compressor + var comp encoding.Compressor + if ct := c.compressorType; ct != "" { + callHdr.SendCompress = ct + if ct != encoding.Identity { + comp = encoding.GetCompressor(ct) + if comp == nil { + return nil, status.Errorf(codes.Internal, "grpc: Compressor is not installed for requested grpc-encoding %q", ct) + } + } + } else if cc.dopts.cp != nil { callHdr.SendCompress = cc.dopts.cp.Type() + cp = cc.dopts.cp + } + if c.creds != nil { + callHdr.Creds = c.creds } var trInfo traceInfo if EnableTracing { @@ -144,333 +215,400 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth } }() } - if stats.On() { - ctx = stats.TagRPC(ctx, &stats.RPCTagInfo{FullMethodName: method}) + ctx = newContextWithRPCInfo(ctx, c.failFast) + sh := cc.dopts.copts.StatsHandler + var beginTime time.Time + if sh != nil { + ctx = sh.TagRPC(ctx, &stats.RPCTagInfo{FullMethodName: method, FailFast: c.failFast}) + beginTime = time.Now() begin := &stats.Begin{ Client: true, - BeginTime: time.Now(), + BeginTime: beginTime, FailFast: c.failFast, } - stats.HandleRPC(ctx, begin) - } - defer func() { - if err != nil && stats.On() { - // Only handle end stats if err != nil. - end := &stats.End{ - Client: true, - Error: err, + sh.HandleRPC(ctx, begin) + defer func() { + if err != nil { + // Only handle end stats if err != nil. + end := &stats.End{ + Client: true, + Error: err, + BeginTime: beginTime, + EndTime: time.Now(), + } + sh.HandleRPC(ctx, end) } - stats.HandleRPC(ctx, end) - } - }() - gopts := BalancerGetOptions{ - BlockingWait: !c.failFast, + }() } + + var ( + t transport.ClientTransport + s *transport.Stream + done func(balancer.DoneInfo) + ) for { - t, put, err = cc.getTransport(ctx, gopts) + // Check to make sure the context has expired. This will prevent us from + // looping forever if an error occurs for wait-for-ready RPCs where no data + // is sent on the wire. + select { + case <-ctx.Done(): + return nil, toRPCErr(ctx.Err()) + default: + } + + t, done, err = cc.getTransport(ctx, c.failFast) if err != nil { - // TODO(zhaoq): Probably revisit the error handling. - if _, ok := err.(*rpcError); ok { - return nil, err - } - if err == errConnClosing || err == errConnUnavailable { - if c.failFast { - return nil, Errorf(codes.Unavailable, "%v", err) - } - continue - } - // All the other errors are treated as Internal errors. - return nil, Errorf(codes.Internal, "%v", err) + return nil, err } s, err = t.NewStream(ctx, callHdr) if err != nil { - if put != nil { - put() - put = nil + if done != nil { + done(balancer.DoneInfo{Err: err}) + done = nil } - if _, ok := err.(transport.ConnectionError); ok || err == transport.ErrStreamDrain { - if c.failFast { - return nil, toRPCErr(err) - } + // In the event of any error from NewStream, we never attempted to write + // anything to the wire, so we can retry indefinitely for non-fail-fast + // RPCs. + if !c.failFast { continue } return nil, toRPCErr(err) } break } - cs := &clientStream{ - opts: opts, - c: c, - desc: desc, - codec: cc.dopts.codec, - cp: cc.dopts.cp, - dc: cc.dopts.dc, - - put: put, - t: t, - s: s, - p: &parser{r: s}, - - tracing: EnableTracing, - trInfo: trInfo, - statsCtx: ctx, - } - if cc.dopts.cp != nil { - cs.cbuf = new(bytes.Buffer) - } - // Listen on ctx.Done() to detect cancellation and s.Done() to detect normal termination - // when there is no pending I/O operations on this stream. - go func() { - select { - case <-t.Error(): - // Incur transport error, simply exit. - case <-s.Done(): - // TODO: The trace of the RPC is terminated here when there is no pending - // I/O, which is probably not the optimal solution. - if s.StatusCode() == codes.OK { - cs.finish(nil) - } else { - cs.finish(Errorf(s.StatusCode(), "%s", s.StatusDesc())) + cs := &clientStream{ + opts: opts, + c: c, + cc: cc, + desc: desc, + codec: c.codec, + cp: cp, + comp: comp, + cancel: cancel, + attempt: &csAttempt{ + t: t, + s: s, + p: &parser{r: s}, + done: done, + dc: cc.dopts.dc, + ctx: ctx, + trInfo: trInfo, + statsHandler: sh, + beginTime: beginTime, + }, + } + cs.c.stream = cs + cs.attempt.cs = cs + if desc != unaryStreamDesc { + // Listen on cc and stream contexts to cleanup when the user closes the + // ClientConn or cancels the stream context. In all other cases, an error + // should already be injected into the recv buffer by the transport, which + // the client will eventually receive, and then we will cancel the stream's + // context in clientStream.finish. + go func() { + select { + case <-cc.ctx.Done(): + cs.finish(ErrClientConnClosing) + case <-ctx.Done(): + cs.finish(toRPCErr(ctx.Err())) } - cs.closeTransportStream(nil) - case <-s.GoAway(): - cs.finish(errConnDrain) - cs.closeTransportStream(errConnDrain) - case <-s.Context().Done(): - err := s.Context().Err() - cs.finish(err) - cs.closeTransportStream(transport.ContextErr(err)) - } - }() + }() + } return cs, nil } // clientStream implements a client side Stream. type clientStream struct { - opts []CallOption - c callInfo - t transport.ClientTransport - s *transport.Stream - p *parser - desc *StreamDesc - codec Codec + opts []CallOption + c *callInfo + cc *ClientConn + desc *StreamDesc + + codec baseCodec cp Compressor - cbuf *bytes.Buffer - dc Decompressor + comp encoding.Compressor - tracing bool // set to EnableTracing when the clientStream is created. + cancel context.CancelFunc // cancels all attempts - mu sync.Mutex - put func() - closed bool - // trInfo.tr is set when the clientStream is created (if EnableTracing is true), - // and is set to nil when the clientStream's finish method is called. + sentLast bool // sent an end stream + + mu sync.Mutex // guards finished + finished bool // TODO: replace with atomic cmpxchg or sync.Once? + + attempt *csAttempt // the active client stream attempt + // TODO(hedging): hedging will have multiple attempts simultaneously. +} + +// csAttempt implements a single transport stream attempt within a +// clientStream. +type csAttempt struct { + cs *clientStream + t transport.ClientTransport + s *transport.Stream + p *parser + done func(balancer.DoneInfo) + + dc Decompressor + decomp encoding.Compressor + decompSet bool + + ctx context.Context // the application's context, wrapped by stats/tracing + + mu sync.Mutex // guards trInfo.tr + // trInfo.tr is set when created (if EnableTracing is true), + // and cleared when the finish method is called. trInfo traceInfo - // statsCtx keeps the user context for stats handling. - // All stats collection should use the statsCtx (instead of the stream context) - // so that all the generated stats for a particular RPC can be associated in the processing phase. - statsCtx context.Context + statsHandler stats.Handler + beginTime time.Time } func (cs *clientStream) Context() context.Context { - return cs.s.Context() + // TODO(retry): commit the current attempt (the context has peer-aware data). + return cs.attempt.context() } func (cs *clientStream) Header() (metadata.MD, error) { - m, err := cs.s.Header() + m, err := cs.attempt.header() if err != nil { - if _, ok := err.(transport.ConnectionError); !ok { - cs.closeTransportStream(err) - } + // TODO(retry): maybe retry on error or commit attempt on success. + err = toRPCErr(err) + cs.finish(err) } return m, err } func (cs *clientStream) Trailer() metadata.MD { - return cs.s.Trailer() + // TODO(retry): on error, maybe retry (trailers-only). + return cs.attempt.trailer() } func (cs *clientStream) SendMsg(m interface{}) (err error) { - if cs.tracing { - cs.mu.Lock() - if cs.trInfo.tr != nil { - cs.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true) - } + // TODO(retry): buffer message for replaying if not committed. + return cs.attempt.sendMsg(m) +} + +func (cs *clientStream) RecvMsg(m interface{}) (err error) { + // TODO(retry): maybe retry on error or commit attempt on success. + return cs.attempt.recvMsg(m) +} + +func (cs *clientStream) CloseSend() error { + cs.attempt.closeSend() + return nil +} + +func (cs *clientStream) finish(err error) { + if err == io.EOF { + // Ending a stream with EOF indicates a success. + err = nil + } + cs.mu.Lock() + if cs.finished { cs.mu.Unlock() + return + } + cs.finished = true + cs.mu.Unlock() + if channelz.IsOn() { + if err != nil { + cs.cc.incrCallsFailed() + } else { + cs.cc.incrCallsSucceeded() + } } + // TODO(retry): commit current attempt if necessary. + cs.attempt.finish(err) + for _, o := range cs.opts { + o.after(cs.c) + } + cs.cancel() +} + +func (a *csAttempt) context() context.Context { + return a.s.Context() +} + +func (a *csAttempt) header() (metadata.MD, error) { + return a.s.Header() +} + +func (a *csAttempt) trailer() metadata.MD { + return a.s.Trailer() +} + +func (a *csAttempt) sendMsg(m interface{}) (err error) { // TODO Investigate how to signal the stats handling party. // generate error stats if err != nil && err != io.EOF? + cs := a.cs defer func() { - if err != nil { - cs.finish(err) - } - if err == nil { - return - } - if err == io.EOF { - // Specialize the process for server streaming. SendMesg is only called - // once when creating the stream object. io.EOF needs to be skipped when - // the rpc is early finished (before the stream object is created.). - // TODO: It is probably better to move this into the generated code. - if !cs.desc.ClientStreams && cs.desc.ServerStreams { - err = nil - } - return + // For non-client-streaming RPCs, we return nil instead of EOF on success + // because the generated code requires it. finish is not called; RecvMsg() + // will call it with the stream's status independently. + if err == io.EOF && !cs.desc.ClientStreams { + err = nil } - if _, ok := err.(transport.ConnectionError); !ok { - cs.closeTransportStream(err) + if err != nil && err != io.EOF { + // Call finish on the client stream for errors generated by this SendMsg + // call, as these indicate problems created by this client. (Transport + // errors are converted to an io.EOF error below; the real error will be + // returned from RecvMsg eventually in that case, or be retried.) + cs.finish(err) } - err = toRPCErr(err) }() + // TODO: Check cs.sentLast and error if we already ended the stream. + if EnableTracing { + a.mu.Lock() + if a.trInfo.tr != nil { + a.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true) + } + a.mu.Unlock() + } var outPayload *stats.OutPayload - if stats.On() { + if a.statsHandler != nil { outPayload = &stats.OutPayload{ Client: true, } } - out, err := encode(cs.codec, m, cs.cp, cs.cbuf, outPayload) - defer func() { - if cs.cbuf != nil { - cs.cbuf.Reset() - } - }() + hdr, data, err := encode(cs.codec, m, cs.cp, outPayload, cs.comp) if err != nil { - return Errorf(codes.Internal, "grpc: %v", err) + return err } - err = cs.t.Write(cs.s, out, &transport.Options{Last: false}) - if err == nil && outPayload != nil { - outPayload.SentTime = time.Now() - stats.HandleRPC(cs.statsCtx, outPayload) + if len(data) > *cs.c.maxSendMessageSize { + return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(data), *cs.c.maxSendMessageSize) + } + if !cs.desc.ClientStreams { + cs.sentLast = true } - return err + err = a.t.Write(a.s, hdr, data, &transport.Options{Last: !cs.desc.ClientStreams}) + if err == nil { + if outPayload != nil { + outPayload.SentTime = time.Now() + a.statsHandler.HandleRPC(a.ctx, outPayload) + } + if channelz.IsOn() { + a.t.IncrMsgSent() + } + return nil + } + return io.EOF } -func (cs *clientStream) RecvMsg(m interface{}) (err error) { +func (a *csAttempt) recvMsg(m interface{}) (err error) { + cs := a.cs defer func() { - if err != nil && stats.On() { - // Only generate End if err != nil. - // If err == nil, it's not the last RecvMsg. - // The last RecvMsg gets either an RPC error or io.EOF. - end := &stats.End{ - Client: true, - EndTime: time.Now(), - } - if err != io.EOF { - end.Error = toRPCErr(err) - } - stats.HandleRPC(cs.statsCtx, end) + if err != nil || !cs.desc.ServerStreams { + // err != nil or non-server-streaming indicates end of stream. + cs.finish(err) } }() var inPayload *stats.InPayload - if stats.On() { + if a.statsHandler != nil { inPayload = &stats.InPayload{ Client: true, } } - err = recv(cs.p, cs.codec, cs.s, cs.dc, m, math.MaxInt32, inPayload) - defer func() { - // err != nil indicates the termination of the stream. - if err != nil { - cs.finish(err) - } - }() - if err == nil { - if cs.tracing { - cs.mu.Lock() - if cs.trInfo.tr != nil { - cs.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true) + if !a.decompSet { + // Block until we receive headers containing received message encoding. + if ct := a.s.RecvCompress(); ct != "" && ct != encoding.Identity { + if a.dc == nil || a.dc.Type() != ct { + // No configured decompressor, or it does not match the incoming + // message encoding; attempt to find a registered compressor that does. + a.dc = nil + a.decomp = encoding.GetCompressor(ct) } - cs.mu.Unlock() - } - if inPayload != nil { - stats.HandleRPC(cs.statsCtx, inPayload) - } - if !cs.desc.ClientStreams || cs.desc.ServerStreams { - return - } - // Special handling for client streaming rpc. - // This recv expects EOF or errors, so we don't collect inPayload. - err = recv(cs.p, cs.codec, cs.s, cs.dc, m, math.MaxInt32, nil) - cs.closeTransportStream(err) - if err == nil { - return toRPCErr(errors.New("grpc: client streaming protocol violation: get , want ")) + } else { + // No compression is used; disable our decompressor. + a.dc = nil } + // Only initialize this state once per stream. + a.decompSet = true + } + err = recv(a.p, cs.codec, a.s, a.dc, m, *cs.c.maxReceiveMessageSize, inPayload, a.decomp) + if err != nil { if err == io.EOF { - if cs.s.StatusCode() == codes.OK { - cs.finish(err) - return nil + if statusErr := a.s.Status().Err(); statusErr != nil { + return statusErr } - return Errorf(cs.s.StatusCode(), "%s", cs.s.StatusDesc()) + return io.EOF // indicates successful end of stream. } return toRPCErr(err) } - if _, ok := err.(transport.ConnectionError); !ok { - cs.closeTransportStream(err) - } - if err == io.EOF { - if cs.s.StatusCode() == codes.OK { - // Returns io.EOF to indicate the end of the stream. - return + if EnableTracing { + a.mu.Lock() + if a.trInfo.tr != nil { + a.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true) } - return Errorf(cs.s.StatusCode(), "%s", cs.s.StatusDesc()) + a.mu.Unlock() } - return toRPCErr(err) -} - -func (cs *clientStream) CloseSend() (err error) { - err = cs.t.Write(cs.s, nil, &transport.Options{Last: true}) - defer func() { - if err != nil { - cs.finish(err) - } - }() - if err == nil || err == io.EOF { + if inPayload != nil { + a.statsHandler.HandleRPC(a.ctx, inPayload) + } + if channelz.IsOn() { + a.t.IncrMsgRecv() + } + if cs.desc.ServerStreams { + // Subsequent messages should be received by subsequent RecvMsg calls. return nil } - if _, ok := err.(transport.ConnectionError); !ok { - cs.closeTransportStream(err) + + // Special handling for non-server-stream rpcs. + // This recv expects EOF or errors, so we don't collect inPayload. + err = recv(a.p, cs.codec, a.s, a.dc, m, *cs.c.maxReceiveMessageSize, nil, a.decomp) + if err == nil { + return toRPCErr(errors.New("grpc: client streaming protocol violation: get , want ")) + } + if err == io.EOF { + return a.s.Status().Err() // non-server streaming Recv returns nil on success } - err = toRPCErr(err) - return + return toRPCErr(err) } -func (cs *clientStream) closeTransportStream(err error) { - cs.mu.Lock() - if cs.closed { - cs.mu.Unlock() +func (a *csAttempt) closeSend() { + cs := a.cs + if cs.sentLast { return } - cs.closed = true - cs.mu.Unlock() - cs.t.CloseStream(cs.s, err) + cs.sentLast = true + cs.attempt.t.Write(cs.attempt.s, nil, nil, &transport.Options{Last: true}) + // We ignore errors from Write. Any error it would return would also be + // returned by a subsequent RecvMsg call, and the user is supposed to always + // finish the stream by calling RecvMsg until it returns err != nil. } -func (cs *clientStream) finish(err error) { - cs.mu.Lock() - defer cs.mu.Unlock() - for _, o := range cs.opts { - o.after(&cs.c) - } - if cs.put != nil { - cs.put() - cs.put = nil +func (a *csAttempt) finish(err error) { + a.mu.Lock() + a.t.CloseStream(a.s, err) + + if a.done != nil { + a.done(balancer.DoneInfo{ + Err: err, + BytesSent: true, + BytesReceived: a.s.BytesReceived(), + }) } - if !cs.tracing { - return + if a.statsHandler != nil { + end := &stats.End{ + Client: true, + BeginTime: a.beginTime, + EndTime: time.Now(), + Error: err, + } + a.statsHandler.HandleRPC(a.ctx, end) } - if cs.trInfo.tr != nil { - if err == nil || err == io.EOF { - cs.trInfo.tr.LazyPrintf("RPC: [OK]") + if a.trInfo.tr != nil { + if err == nil { + a.trInfo.tr.LazyPrintf("RPC: [OK]") } else { - cs.trInfo.tr.LazyPrintf("RPC: [%v]", err) - cs.trInfo.tr.SetError() + a.trInfo.tr.LazyPrintf("RPC: [%v]", err) + a.trInfo.tr.SetError() } - cs.trInfo.tr.Finish() - cs.trInfo.tr = nil + a.trInfo.tr.Finish() + a.trInfo.tr = nil } + a.mu.Unlock() } // ServerStream defines the interface a server stream has to satisfy. @@ -494,23 +632,28 @@ type ServerStream interface { // serverStream implements a server side Stream. type serverStream struct { - t transport.ServerTransport - s *transport.Stream - p *parser - codec Codec - cp Compressor - dc Decompressor - cbuf *bytes.Buffer - maxMsgSize int - statusCode codes.Code - statusDesc string - trInfo *traceInfo + ctx context.Context + t transport.ServerTransport + s *transport.Stream + p *parser + codec baseCodec + + cp Compressor + dc Decompressor + comp encoding.Compressor + decomp encoding.Compressor + + maxReceiveMessageSize int + maxSendMessageSize int + trInfo *traceInfo + + statsHandler stats.Handler mu sync.Mutex // protects trInfo.tr after the service handler runs. } func (ss *serverStream) Context() context.Context { - return ss.s.Context() + return ss.ctx } func (ss *serverStream) SetHeader(md metadata.MD) error { @@ -529,7 +672,6 @@ func (ss *serverStream) SetTrailer(md metadata.MD) { return } ss.s.SetTrailer(md) - return } func (ss *serverStream) SendMsg(m interface{}) (err error) { @@ -546,27 +688,31 @@ func (ss *serverStream) SendMsg(m interface{}) (err error) { } ss.mu.Unlock() } + if err != nil && err != io.EOF { + st, _ := status.FromError(toRPCErr(err)) + ss.t.WriteStatus(ss.s, st) + } + if channelz.IsOn() && err == nil { + ss.t.IncrMsgSent() + } }() var outPayload *stats.OutPayload - if stats.On() { + if ss.statsHandler != nil { outPayload = &stats.OutPayload{} } - out, err := encode(ss.codec, m, ss.cp, ss.cbuf, outPayload) - defer func() { - if ss.cbuf != nil { - ss.cbuf.Reset() - } - }() + hdr, data, err := encode(ss.codec, m, ss.cp, outPayload, ss.comp) if err != nil { - err = Errorf(codes.Internal, "grpc: %v", err) return err } - if err := ss.t.Write(ss.s, out, &transport.Options{Last: false}); err != nil { + if len(data) > ss.maxSendMessageSize { + return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(data), ss.maxSendMessageSize) + } + if err := ss.t.Write(ss.s, hdr, data, &transport.Options{Last: false}); err != nil { return toRPCErr(err) } if outPayload != nil { outPayload.SentTime = time.Now() - stats.HandleRPC(ss.s.Context(), outPayload) + ss.statsHandler.HandleRPC(ss.s.Context(), outPayload) } return nil } @@ -585,22 +731,35 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { } ss.mu.Unlock() } + if err != nil && err != io.EOF { + st, _ := status.FromError(toRPCErr(err)) + ss.t.WriteStatus(ss.s, st) + } + if channelz.IsOn() && err == nil { + ss.t.IncrMsgRecv() + } }() var inPayload *stats.InPayload - if stats.On() { + if ss.statsHandler != nil { inPayload = &stats.InPayload{} } - if err := recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxMsgSize, inPayload); err != nil { + if err := recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxReceiveMessageSize, inPayload, ss.decomp); err != nil { if err == io.EOF { return err } if err == io.ErrUnexpectedEOF { - err = Errorf(codes.Internal, io.ErrUnexpectedEOF.Error()) + err = status.Errorf(codes.Internal, io.ErrUnexpectedEOF.Error()) } return toRPCErr(err) } if inPayload != nil { - stats.HandleRPC(ss.s.Context(), inPayload) + ss.statsHandler.HandleRPC(ss.s.Context(), inPayload) } return nil } + +// MethodFromServerStream returns the method string for the input stream. +// The returned string is in the format of "/service/method". +func MethodFromServerStream(stream ServerStream) (string, bool) { + return Method(stream.Context()) +} diff --git a/vendor/google.golang.org/grpc/trace.go b/vendor/google.golang.org/grpc/trace.go index f6747e1dfa4..c1c96dedcb7 100644 --- a/vendor/google.golang.org/grpc/trace.go +++ b/vendor/google.golang.org/grpc/trace.go @@ -1,33 +1,18 @@ /* * - * Copyright 2015, Google Inc. - * All rights reserved. + * Copyright 2015 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -46,7 +31,7 @@ import ( // EnableTracing controls whether to trace RPCs using the golang.org/x/net/trace package. // This should only be set before any RPCs are sent or received by this program. -var EnableTracing = true +var EnableTracing bool // methodFamily returns the trace family for the given method. // It turns "/pkg.Service/GetFoo" into "pkg.Service". @@ -91,6 +76,15 @@ func (f *firstLine) String() string { return line.String() } +const truncateSize = 100 + +func truncate(x string, l int) string { + if l > len(x) { + return x + } + return x[:l] +} + // payload represents an RPC request or response payload. type payload struct { sent bool // whether this is an outgoing payload @@ -100,9 +94,9 @@ type payload struct { func (p payload) String() string { if p.sent { - return fmt.Sprintf("sent: %v", p.msg) + return truncate(fmt.Sprintf("sent: %v", p.msg), truncateSize) } - return fmt.Sprintf("recv: %v", p.msg) + return truncate(fmt.Sprintf("recv: %v", p.msg), truncateSize) } type fmtStringer struct { diff --git a/vendor/google.golang.org/grpc/transport/bdp_estimator.go b/vendor/google.golang.org/grpc/transport/bdp_estimator.go new file mode 100644 index 00000000000..63cd2627c87 --- /dev/null +++ b/vendor/google.golang.org/grpc/transport/bdp_estimator.go @@ -0,0 +1,140 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "sync" + "time" +) + +const ( + // bdpLimit is the maximum value the flow control windows + // will be increased to. + bdpLimit = (1 << 20) * 4 + // alpha is a constant factor used to keep a moving average + // of RTTs. + alpha = 0.9 + // If the current bdp sample is greater than or equal to + // our beta * our estimated bdp and the current bandwidth + // sample is the maximum bandwidth observed so far, we + // increase our bbp estimate by a factor of gamma. + beta = 0.66 + // To put our bdp to be smaller than or equal to twice the real BDP, + // we should multiply our current sample with 4/3, however to round things out + // we use 2 as the multiplication factor. + gamma = 2 +) + +// Adding arbitrary data to ping so that its ack can be identified. +// Easter-egg: what does the ping message say? +var bdpPing = &ping{data: [8]byte{2, 4, 16, 16, 9, 14, 7, 7}} + +type bdpEstimator struct { + // sentAt is the time when the ping was sent. + sentAt time.Time + + mu sync.Mutex + // bdp is the current bdp estimate. + bdp uint32 + // sample is the number of bytes received in one measurement cycle. + sample uint32 + // bwMax is the maximum bandwidth noted so far (bytes/sec). + bwMax float64 + // bool to keep track of the beginning of a new measurement cycle. + isSent bool + // Callback to update the window sizes. + updateFlowControl func(n uint32) + // sampleCount is the number of samples taken so far. + sampleCount uint64 + // round trip time (seconds) + rtt float64 +} + +// timesnap registers the time bdp ping was sent out so that +// network rtt can be calculated when its ack is received. +// It is called (by controller) when the bdpPing is +// being written on the wire. +func (b *bdpEstimator) timesnap(d [8]byte) { + if bdpPing.data != d { + return + } + b.sentAt = time.Now() +} + +// add adds bytes to the current sample for calculating bdp. +// It returns true only if a ping must be sent. This can be used +// by the caller (handleData) to make decision about batching +// a window update with it. +func (b *bdpEstimator) add(n uint32) bool { + b.mu.Lock() + defer b.mu.Unlock() + if b.bdp == bdpLimit { + return false + } + if !b.isSent { + b.isSent = true + b.sample = n + b.sentAt = time.Time{} + b.sampleCount++ + return true + } + b.sample += n + return false +} + +// calculate is called when an ack for a bdp ping is received. +// Here we calculate the current bdp and bandwidth sample and +// decide if the flow control windows should go up. +func (b *bdpEstimator) calculate(d [8]byte) { + // Check if the ping acked for was the bdp ping. + if bdpPing.data != d { + return + } + b.mu.Lock() + rttSample := time.Since(b.sentAt).Seconds() + if b.sampleCount < 10 { + // Bootstrap rtt with an average of first 10 rtt samples. + b.rtt += (rttSample - b.rtt) / float64(b.sampleCount) + } else { + // Heed to the recent past more. + b.rtt += (rttSample - b.rtt) * float64(alpha) + } + b.isSent = false + // The number of bytes accumulated so far in the sample is smaller + // than or equal to 1.5 times the real BDP on a saturated connection. + bwCurrent := float64(b.sample) / (b.rtt * float64(1.5)) + if bwCurrent > b.bwMax { + b.bwMax = bwCurrent + } + // If the current sample (which is smaller than or equal to the 1.5 times the real BDP) is + // greater than or equal to 2/3rd our perceived bdp AND this is the maximum bandwidth seen so far, we + // should update our perception of the network BDP. + if float64(b.sample) >= beta*float64(b.bdp) && bwCurrent == b.bwMax && b.bdp != bdpLimit { + sampleFloat := float64(b.sample) + b.bdp = uint32(gamma * sampleFloat) + if b.bdp > bdpLimit { + b.bdp = bdpLimit + } + bdp := b.bdp + b.mu.Unlock() + b.updateFlowControl(bdp) + return + } + b.mu.Unlock() +} diff --git a/vendor/google.golang.org/grpc/transport/control.go b/vendor/google.golang.org/grpc/transport/control.go deleted file mode 100644 index 2586cba469c..00000000000 --- a/vendor/google.golang.org/grpc/transport/control.go +++ /dev/null @@ -1,193 +0,0 @@ -/* - * - * Copyright 2014, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -package transport - -import ( - "fmt" - "sync" - - "golang.org/x/net/http2" -) - -const ( - // The default value of flow control window size in HTTP2 spec. - defaultWindowSize = 65535 - // The initial window size for flow control. - initialWindowSize = defaultWindowSize // for an RPC - initialConnWindowSize = defaultWindowSize * 16 // for a connection -) - -// The following defines various control items which could flow through -// the control buffer of transport. They represent different aspects of -// control tasks, e.g., flow control, settings, streaming resetting, etc. -type windowUpdate struct { - streamID uint32 - increment uint32 -} - -func (*windowUpdate) item() {} - -type settings struct { - ack bool - ss []http2.Setting -} - -func (*settings) item() {} - -type resetStream struct { - streamID uint32 - code http2.ErrCode -} - -func (*resetStream) item() {} - -type goAway struct { -} - -func (*goAway) item() {} - -type flushIO struct { -} - -func (*flushIO) item() {} - -type ping struct { - ack bool - data [8]byte -} - -func (*ping) item() {} - -// quotaPool is a pool which accumulates the quota and sends it to acquire() -// when it is available. -type quotaPool struct { - c chan int - - mu sync.Mutex - quota int -} - -// newQuotaPool creates a quotaPool which has quota q available to consume. -func newQuotaPool(q int) *quotaPool { - qb := "aPool{ - c: make(chan int, 1), - } - if q > 0 { - qb.c <- q - } else { - qb.quota = q - } - return qb -} - -// add cancels the pending quota sent on acquired, incremented by v and sends -// it back on acquire. -func (qb *quotaPool) add(v int) { - qb.mu.Lock() - defer qb.mu.Unlock() - select { - case n := <-qb.c: - qb.quota += n - default: - } - qb.quota += v - if qb.quota <= 0 { - return - } - // After the pool has been created, this is the only place that sends on - // the channel. Since mu is held at this point and any quota that was sent - // on the channel has been retrieved, we know that this code will always - // place any positive quota value on the channel. - select { - case qb.c <- qb.quota: - qb.quota = 0 - default: - } -} - -// acquire returns the channel on which available quota amounts are sent. -func (qb *quotaPool) acquire() <-chan int { - return qb.c -} - -// inFlow deals with inbound flow control -type inFlow struct { - // The inbound flow control limit for pending data. - limit uint32 - - mu sync.Mutex - // pendingData is the overall data which have been received but not been - // consumed by applications. - pendingData uint32 - // The amount of data the application has consumed but grpc has not sent - // window update for them. Used to reduce window update frequency. - pendingUpdate uint32 -} - -// onData is invoked when some data frame is received. It updates pendingData. -func (f *inFlow) onData(n uint32) error { - f.mu.Lock() - defer f.mu.Unlock() - f.pendingData += n - if f.pendingData+f.pendingUpdate > f.limit { - return fmt.Errorf("received %d-bytes data exceeding the limit %d bytes", f.pendingData+f.pendingUpdate, f.limit) - } - return nil -} - -// onRead is invoked when the application reads the data. It returns the window size -// to be sent to the peer. -func (f *inFlow) onRead(n uint32) uint32 { - f.mu.Lock() - defer f.mu.Unlock() - if f.pendingData == 0 { - return 0 - } - f.pendingData -= n - f.pendingUpdate += n - if f.pendingUpdate >= f.limit/4 { - wu := f.pendingUpdate - f.pendingUpdate = 0 - return wu - } - return 0 -} - -func (f *inFlow) resetPendingData() uint32 { - f.mu.Lock() - defer f.mu.Unlock() - n := f.pendingData - f.pendingData = 0 - return n -} diff --git a/vendor/google.golang.org/grpc/transport/controlbuf.go b/vendor/google.golang.org/grpc/transport/controlbuf.go new file mode 100644 index 00000000000..e147cd51bf1 --- /dev/null +++ b/vendor/google.golang.org/grpc/transport/controlbuf.go @@ -0,0 +1,769 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "bytes" + "fmt" + "runtime" + "sync" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" +) + +type itemNode struct { + it interface{} + next *itemNode +} + +type itemList struct { + head *itemNode + tail *itemNode +} + +func (il *itemList) enqueue(i interface{}) { + n := &itemNode{it: i} + if il.tail == nil { + il.head, il.tail = n, n + return + } + il.tail.next = n + il.tail = n +} + +// peek returns the first item in the list without removing it from the +// list. +func (il *itemList) peek() interface{} { + return il.head.it +} + +func (il *itemList) dequeue() interface{} { + if il.head == nil { + return nil + } + i := il.head.it + il.head = il.head.next + if il.head == nil { + il.tail = nil + } + return i +} + +func (il *itemList) dequeueAll() *itemNode { + h := il.head + il.head, il.tail = nil, nil + return h +} + +func (il *itemList) isEmpty() bool { + return il.head == nil +} + +// The following defines various control items which could flow through +// the control buffer of transport. They represent different aspects of +// control tasks, e.g., flow control, settings, streaming resetting, etc. + +type headerFrame struct { + streamID uint32 + hf []hpack.HeaderField + endStream bool // Valid on server side. + initStream func(uint32) (bool, error) // Used only on the client side. + onWrite func() + wq *writeQuota // write quota for the stream created. + cleanup *cleanupStream // Valid on the server side. + onOrphaned func(error) // Valid on client-side +} + +type cleanupStream struct { + streamID uint32 + idPtr *uint32 + rst bool + rstCode http2.ErrCode + onWrite func() +} + +type dataFrame struct { + streamID uint32 + endStream bool + h []byte + d []byte + // onEachWrite is called every time + // a part of d is written out. + onEachWrite func() +} + +type incomingWindowUpdate struct { + streamID uint32 + increment uint32 +} + +type outgoingWindowUpdate struct { + streamID uint32 + increment uint32 +} + +type incomingSettings struct { + ss []http2.Setting +} + +type outgoingSettings struct { + ss []http2.Setting +} + +type settingsAck struct { +} + +type incomingGoAway struct { +} + +type goAway struct { + code http2.ErrCode + debugData []byte + headsUp bool + closeConn bool +} + +type ping struct { + ack bool + data [8]byte +} + +type outFlowControlSizeRequest struct { + resp chan uint32 +} + +type outStreamState int + +const ( + active outStreamState = iota + empty + waitingOnStreamQuota +) + +type outStream struct { + id uint32 + state outStreamState + itl *itemList + bytesOutStanding int + wq *writeQuota + + next *outStream + prev *outStream +} + +func (s *outStream) deleteSelf() { + if s.prev != nil { + s.prev.next = s.next + } + if s.next != nil { + s.next.prev = s.prev + } + s.next, s.prev = nil, nil +} + +type outStreamList struct { + // Following are sentinel objects that mark the + // beginning and end of the list. They do not + // contain any item lists. All valid objects are + // inserted in between them. + // This is needed so that an outStream object can + // deleteSelf() in O(1) time without knowing which + // list it belongs to. + head *outStream + tail *outStream +} + +func newOutStreamList() *outStreamList { + head, tail := new(outStream), new(outStream) + head.next = tail + tail.prev = head + return &outStreamList{ + head: head, + tail: tail, + } +} + +func (l *outStreamList) enqueue(s *outStream) { + e := l.tail.prev + e.next = s + s.prev = e + s.next = l.tail + l.tail.prev = s +} + +// remove from the beginning of the list. +func (l *outStreamList) dequeue() *outStream { + b := l.head.next + if b == l.tail { + return nil + } + b.deleteSelf() + return b +} + +type controlBuffer struct { + ch chan struct{} + done <-chan struct{} + mu sync.Mutex + consumerWaiting bool + list *itemList + err error +} + +func newControlBuffer(done <-chan struct{}) *controlBuffer { + return &controlBuffer{ + ch: make(chan struct{}, 1), + list: &itemList{}, + done: done, + } +} + +func (c *controlBuffer) put(it interface{}) error { + _, err := c.executeAndPut(nil, it) + return err +} + +func (c *controlBuffer) executeAndPut(f func(it interface{}) bool, it interface{}) (bool, error) { + var wakeUp bool + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return false, c.err + } + if f != nil { + if !f(it) { // f wasn't successful + c.mu.Unlock() + return false, nil + } + } + if c.consumerWaiting { + wakeUp = true + c.consumerWaiting = false + } + c.list.enqueue(it) + c.mu.Unlock() + if wakeUp { + select { + case c.ch <- struct{}{}: + default: + } + } + return true, nil +} + +func (c *controlBuffer) get(block bool) (interface{}, error) { + for { + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return nil, c.err + } + if !c.list.isEmpty() { + h := c.list.dequeue() + c.mu.Unlock() + return h, nil + } + if !block { + c.mu.Unlock() + return nil, nil + } + c.consumerWaiting = true + c.mu.Unlock() + select { + case <-c.ch: + case <-c.done: + c.finish() + return nil, ErrConnClosing + } + } +} + +func (c *controlBuffer) finish() { + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return + } + c.err = ErrConnClosing + // There may be headers for streams in the control buffer. + // These streams need to be cleaned out since the transport + // is still not aware of these yet. + for head := c.list.dequeueAll(); head != nil; head = head.next { + hdr, ok := head.it.(*headerFrame) + if !ok { + continue + } + if hdr.onOrphaned != nil { // It will be nil on the server-side. + hdr.onOrphaned(ErrConnClosing) + } + } + c.mu.Unlock() +} + +type side int + +const ( + clientSide side = iota + serverSide +) + +type loopyWriter struct { + side side + cbuf *controlBuffer + sendQuota uint32 + oiws uint32 // outbound initial window size. + estdStreams map[uint32]*outStream // Established streams. + activeStreams *outStreamList // Streams that are sending data. + framer *framer + hBuf *bytes.Buffer // The buffer for HPACK encoding. + hEnc *hpack.Encoder // HPACK encoder. + bdpEst *bdpEstimator + draining bool + + // Side-specific handlers + ssGoAwayHandler func(*goAway) (bool, error) +} + +func newLoopyWriter(s side, fr *framer, cbuf *controlBuffer, bdpEst *bdpEstimator) *loopyWriter { + var buf bytes.Buffer + l := &loopyWriter{ + side: s, + cbuf: cbuf, + sendQuota: defaultWindowSize, + oiws: defaultWindowSize, + estdStreams: make(map[uint32]*outStream), + activeStreams: newOutStreamList(), + framer: fr, + hBuf: &buf, + hEnc: hpack.NewEncoder(&buf), + bdpEst: bdpEst, + } + return l +} + +const minBatchSize = 1000 + +// run should be run in a separate goroutine. +func (l *loopyWriter) run() { + var ( + it interface{} + err error + isEmpty bool + ) + defer func() { + errorf("transport: loopyWriter.run returning. Err: %v", err) + }() + for { + it, err = l.cbuf.get(true) + if err != nil { + return + } + if err = l.handle(it); err != nil { + return + } + if _, err = l.processData(); err != nil { + return + } + gosched := true + hasdata: + for { + it, err = l.cbuf.get(false) + if err != nil { + return + } + if it != nil { + if err = l.handle(it); err != nil { + return + } + if _, err = l.processData(); err != nil { + return + } + continue hasdata + } + if isEmpty, err = l.processData(); err != nil { + return + } + if !isEmpty { + continue hasdata + } + if gosched { + gosched = false + if l.framer.writer.offset < minBatchSize { + runtime.Gosched() + continue hasdata + } + } + l.framer.writer.Flush() + break hasdata + + } + } +} + +func (l *loopyWriter) outgoingWindowUpdateHandler(w *outgoingWindowUpdate) error { + return l.framer.fr.WriteWindowUpdate(w.streamID, w.increment) +} + +func (l *loopyWriter) incomingWindowUpdateHandler(w *incomingWindowUpdate) error { + // Otherwise update the quota. + if w.streamID == 0 { + l.sendQuota += w.increment + return nil + } + // Find the stream and update it. + if str, ok := l.estdStreams[w.streamID]; ok { + str.bytesOutStanding -= int(w.increment) + if strQuota := int(l.oiws) - str.bytesOutStanding; strQuota > 0 && str.state == waitingOnStreamQuota { + str.state = active + l.activeStreams.enqueue(str) + return nil + } + } + return nil +} + +func (l *loopyWriter) outgoingSettingsHandler(s *outgoingSettings) error { + return l.framer.fr.WriteSettings(s.ss...) +} + +func (l *loopyWriter) incomingSettingsHandler(s *incomingSettings) error { + if err := l.applySettings(s.ss); err != nil { + return err + } + return l.framer.fr.WriteSettingsAck() +} + +func (l *loopyWriter) headerHandler(h *headerFrame) error { + if l.side == serverSide { + if h.endStream { // Case 1.A: Server wants to close stream. + // Make sure it's not a trailers only response. + if str, ok := l.estdStreams[h.streamID]; ok { + if str.state != empty { // either active or waiting on stream quota. + // add it str's list of items. + str.itl.enqueue(h) + return nil + } + } + if err := l.writeHeader(h.streamID, h.endStream, h.hf, h.onWrite); err != nil { + return err + } + return l.cleanupStreamHandler(h.cleanup) + } + // Case 1.B: Server is responding back with headers. + str := &outStream{ + state: empty, + itl: &itemList{}, + wq: h.wq, + } + l.estdStreams[h.streamID] = str + return l.writeHeader(h.streamID, h.endStream, h.hf, h.onWrite) + } + // Case 2: Client wants to originate stream. + str := &outStream{ + id: h.streamID, + state: empty, + itl: &itemList{}, + wq: h.wq, + } + str.itl.enqueue(h) + return l.originateStream(str) +} + +func (l *loopyWriter) originateStream(str *outStream) error { + hdr := str.itl.dequeue().(*headerFrame) + sendPing, err := hdr.initStream(str.id) + if err != nil { + if err == ErrConnClosing { + return err + } + // Other errors(errStreamDrain) need not close transport. + return nil + } + if err = l.writeHeader(str.id, hdr.endStream, hdr.hf, hdr.onWrite); err != nil { + return err + } + l.estdStreams[str.id] = str + if sendPing { + return l.pingHandler(&ping{data: [8]byte{}}) + } + return nil +} + +func (l *loopyWriter) writeHeader(streamID uint32, endStream bool, hf []hpack.HeaderField, onWrite func()) error { + if onWrite != nil { + onWrite() + } + l.hBuf.Reset() + for _, f := range hf { + if err := l.hEnc.WriteField(f); err != nil { + warningf("transport: loopyWriter.writeHeader encountered error while encoding headers:", err) + } + } + var ( + err error + endHeaders, first bool + ) + first = true + for !endHeaders { + size := l.hBuf.Len() + if size > http2MaxFrameLen { + size = http2MaxFrameLen + } else { + endHeaders = true + } + if first { + first = false + err = l.framer.fr.WriteHeaders(http2.HeadersFrameParam{ + StreamID: streamID, + BlockFragment: l.hBuf.Next(size), + EndStream: endStream, + EndHeaders: endHeaders, + }) + } else { + err = l.framer.fr.WriteContinuation( + streamID, + endHeaders, + l.hBuf.Next(size), + ) + } + if err != nil { + return err + } + } + return nil +} + +func (l *loopyWriter) preprocessData(df *dataFrame) error { + str, ok := l.estdStreams[df.streamID] + if !ok { + return nil + } + // If we got data for a stream it means that + // stream was originated and the headers were sent out. + str.itl.enqueue(df) + if str.state == empty { + str.state = active + l.activeStreams.enqueue(str) + } + return nil +} + +func (l *loopyWriter) pingHandler(p *ping) error { + if !p.ack { + l.bdpEst.timesnap(p.data) + } + return l.framer.fr.WritePing(p.ack, p.data) + +} + +func (l *loopyWriter) outFlowControlSizeRequestHandler(o *outFlowControlSizeRequest) error { + o.resp <- l.sendQuota + return nil +} + +func (l *loopyWriter) cleanupStreamHandler(c *cleanupStream) error { + c.onWrite() + if str, ok := l.estdStreams[c.streamID]; ok { + // On the server side it could be a trailers-only response or + // a RST_STREAM before stream initialization thus the stream might + // not be established yet. + delete(l.estdStreams, c.streamID) + str.deleteSelf() + } + if c.rst { // If RST_STREAM needs to be sent. + if err := l.framer.fr.WriteRSTStream(c.streamID, c.rstCode); err != nil { + return err + } + } + if l.side == clientSide && l.draining && len(l.estdStreams) == 0 { + return ErrConnClosing + } + return nil +} + +func (l *loopyWriter) incomingGoAwayHandler(*incomingGoAway) error { + if l.side == clientSide { + l.draining = true + if len(l.estdStreams) == 0 { + return ErrConnClosing + } + } + return nil +} + +func (l *loopyWriter) goAwayHandler(g *goAway) error { + // Handling of outgoing GoAway is very specific to side. + if l.ssGoAwayHandler != nil { + draining, err := l.ssGoAwayHandler(g) + if err != nil { + return err + } + l.draining = draining + } + return nil +} + +func (l *loopyWriter) handle(i interface{}) error { + switch i := i.(type) { + case *incomingWindowUpdate: + return l.incomingWindowUpdateHandler(i) + case *outgoingWindowUpdate: + return l.outgoingWindowUpdateHandler(i) + case *incomingSettings: + return l.incomingSettingsHandler(i) + case *outgoingSettings: + return l.outgoingSettingsHandler(i) + case *headerFrame: + return l.headerHandler(i) + case *cleanupStream: + return l.cleanupStreamHandler(i) + case *incomingGoAway: + return l.incomingGoAwayHandler(i) + case *dataFrame: + return l.preprocessData(i) + case *ping: + return l.pingHandler(i) + case *goAway: + return l.goAwayHandler(i) + case *outFlowControlSizeRequest: + return l.outFlowControlSizeRequestHandler(i) + default: + return fmt.Errorf("transport: unknown control message type %T", i) + } +} + +func (l *loopyWriter) applySettings(ss []http2.Setting) error { + for _, s := range ss { + switch s.ID { + case http2.SettingInitialWindowSize: + o := l.oiws + l.oiws = s.Val + if o < l.oiws { + // If the new limit is greater make all depleted streams active. + for _, stream := range l.estdStreams { + if stream.state == waitingOnStreamQuota { + stream.state = active + l.activeStreams.enqueue(stream) + } + } + } + } + } + return nil +} + +func (l *loopyWriter) processData() (bool, error) { + if l.sendQuota == 0 { + return true, nil + } + str := l.activeStreams.dequeue() + if str == nil { + return true, nil + } + dataItem := str.itl.peek().(*dataFrame) + if len(dataItem.h) == 0 && len(dataItem.d) == 0 { + // Client sends out empty data frame with endStream = true + if err := l.framer.fr.WriteData(dataItem.streamID, dataItem.endStream, nil); err != nil { + return false, err + } + str.itl.dequeue() + if str.itl.isEmpty() { + str.state = empty + } else if trailer, ok := str.itl.peek().(*headerFrame); ok { // the next item is trailers. + if err := l.writeHeader(trailer.streamID, trailer.endStream, trailer.hf, trailer.onWrite); err != nil { + return false, err + } + if err := l.cleanupStreamHandler(trailer.cleanup); err != nil { + return false, nil + } + } else { + l.activeStreams.enqueue(str) + } + return false, nil + } + var ( + idx int + buf []byte + ) + if len(dataItem.h) != 0 { // data header has not been written out yet. + buf = dataItem.h + } else { + idx = 1 + buf = dataItem.d + } + size := http2MaxFrameLen + if len(buf) < size { + size = len(buf) + } + if strQuota := int(l.oiws) - str.bytesOutStanding; strQuota <= 0 { + str.state = waitingOnStreamQuota + return false, nil + } else if strQuota < size { + size = strQuota + } + + if l.sendQuota < uint32(size) { + size = int(l.sendQuota) + } + // Now that outgoing flow controls are checked we can replenish str's write quota + str.wq.replenish(size) + var endStream bool + // This last data message on this stream and all + // of it can be written in this go. + if dataItem.endStream && size == len(buf) { + // buf contains either data or it contains header but data is empty. + if idx == 1 || len(dataItem.d) == 0 { + endStream = true + } + } + if dataItem.onEachWrite != nil { + dataItem.onEachWrite() + } + if err := l.framer.fr.WriteData(dataItem.streamID, endStream, buf[:size]); err != nil { + return false, err + } + buf = buf[size:] + str.bytesOutStanding += size + l.sendQuota -= uint32(size) + if idx == 0 { + dataItem.h = buf + } else { + dataItem.d = buf + } + + if len(dataItem.h) == 0 && len(dataItem.d) == 0 { // All the data from that message was written out. + str.itl.dequeue() + } + if str.itl.isEmpty() { + str.state = empty + } else if trailer, ok := str.itl.peek().(*headerFrame); ok { // The next item is trailers. + if err := l.writeHeader(trailer.streamID, trailer.endStream, trailer.hf, trailer.onWrite); err != nil { + return false, err + } + if err := l.cleanupStreamHandler(trailer.cleanup); err != nil { + return false, err + } + } else if int(l.oiws)-str.bytesOutStanding <= 0 { // Ran out of stream quota. + str.state = waitingOnStreamQuota + } else { // Otherwise add it back to the list of active streams. + l.activeStreams.enqueue(str) + } + return false, nil +} diff --git a/vendor/google.golang.org/grpc/transport/flowcontrol.go b/vendor/google.golang.org/grpc/transport/flowcontrol.go new file mode 100644 index 00000000000..378f5c4502c --- /dev/null +++ b/vendor/google.golang.org/grpc/transport/flowcontrol.go @@ -0,0 +1,236 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "fmt" + "math" + "sync" + "sync/atomic" + "time" +) + +const ( + // The default value of flow control window size in HTTP2 spec. + defaultWindowSize = 65535 + // The initial window size for flow control. + initialWindowSize = defaultWindowSize // for an RPC + infinity = time.Duration(math.MaxInt64) + defaultClientKeepaliveTime = infinity + defaultClientKeepaliveTimeout = 20 * time.Second + defaultMaxStreamsClient = 100 + defaultMaxConnectionIdle = infinity + defaultMaxConnectionAge = infinity + defaultMaxConnectionAgeGrace = infinity + defaultServerKeepaliveTime = 2 * time.Hour + defaultServerKeepaliveTimeout = 20 * time.Second + defaultKeepalivePolicyMinTime = 5 * time.Minute + // max window limit set by HTTP2 Specs. + maxWindowSize = math.MaxInt32 + // defaultWriteQuota is the default value for number of data + // bytes that each stream can schedule before some of it being + // flushed out. + defaultWriteQuota = 64 * 1024 +) + +// writeQuota is a soft limit on the amount of data a stream can +// schedule before some of it is written out. +type writeQuota struct { + quota int32 + // get waits on read from when quota goes less than or equal to zero. + // replenish writes on it when quota goes positive again. + ch chan struct{} + // done is triggered in error case. + done <-chan struct{} +} + +func newWriteQuota(sz int32, done <-chan struct{}) *writeQuota { + return &writeQuota{ + quota: sz, + ch: make(chan struct{}, 1), + done: done, + } +} + +func (w *writeQuota) get(sz int32) error { + for { + if atomic.LoadInt32(&w.quota) > 0 { + atomic.AddInt32(&w.quota, -sz) + return nil + } + select { + case <-w.ch: + continue + case <-w.done: + return errStreamDone + } + } +} + +func (w *writeQuota) replenish(n int) { + sz := int32(n) + a := atomic.AddInt32(&w.quota, sz) + b := a - sz + if b <= 0 && a > 0 { + select { + case w.ch <- struct{}{}: + default: + } + } +} + +type trInFlow struct { + limit uint32 + unacked uint32 + effectiveWindowSize uint32 +} + +func (f *trInFlow) newLimit(n uint32) uint32 { + d := n - f.limit + f.limit = n + f.updateEffectiveWindowSize() + return d +} + +func (f *trInFlow) onData(n uint32) uint32 { + f.unacked += n + if f.unacked >= f.limit/4 { + w := f.unacked + f.unacked = 0 + f.updateEffectiveWindowSize() + return w + } + f.updateEffectiveWindowSize() + return 0 +} + +func (f *trInFlow) reset() uint32 { + w := f.unacked + f.unacked = 0 + f.updateEffectiveWindowSize() + return w +} + +func (f *trInFlow) updateEffectiveWindowSize() { + atomic.StoreUint32(&f.effectiveWindowSize, f.limit-f.unacked) +} + +func (f *trInFlow) getSize() uint32 { + return atomic.LoadUint32(&f.effectiveWindowSize) +} + +// TODO(mmukhi): Simplify this code. +// inFlow deals with inbound flow control +type inFlow struct { + mu sync.Mutex + // The inbound flow control limit for pending data. + limit uint32 + // pendingData is the overall data which have been received but not been + // consumed by applications. + pendingData uint32 + // The amount of data the application has consumed but grpc has not sent + // window update for them. Used to reduce window update frequency. + pendingUpdate uint32 + // delta is the extra window update given by receiver when an application + // is reading data bigger in size than the inFlow limit. + delta uint32 +} + +// newLimit updates the inflow window to a new value n. +// It assumes that n is always greater than the old limit. +func (f *inFlow) newLimit(n uint32) uint32 { + f.mu.Lock() + d := n - f.limit + f.limit = n + f.mu.Unlock() + return d +} + +func (f *inFlow) maybeAdjust(n uint32) uint32 { + if n > uint32(math.MaxInt32) { + n = uint32(math.MaxInt32) + } + f.mu.Lock() + // estSenderQuota is the receiver's view of the maximum number of bytes the sender + // can send without a window update. + estSenderQuota := int32(f.limit - (f.pendingData + f.pendingUpdate)) + // estUntransmittedData is the maximum number of bytes the sends might not have put + // on the wire yet. A value of 0 or less means that we have already received all or + // more bytes than the application is requesting to read. + estUntransmittedData := int32(n - f.pendingData) // Casting into int32 since it could be negative. + // This implies that unless we send a window update, the sender won't be able to send all the bytes + // for this message. Therefore we must send an update over the limit since there's an active read + // request from the application. + if estUntransmittedData > estSenderQuota { + // Sender's window shouldn't go more than 2^31 - 1 as specified in the HTTP spec. + if f.limit+n > maxWindowSize { + f.delta = maxWindowSize - f.limit + } else { + // Send a window update for the whole message and not just the difference between + // estUntransmittedData and estSenderQuota. This will be helpful in case the message + // is padded; We will fallback on the current available window(at least a 1/4th of the limit). + f.delta = n + } + f.mu.Unlock() + return f.delta + } + f.mu.Unlock() + return 0 +} + +// onData is invoked when some data frame is received. It updates pendingData. +func (f *inFlow) onData(n uint32) error { + f.mu.Lock() + f.pendingData += n + if f.pendingData+f.pendingUpdate > f.limit+f.delta { + limit := f.limit + rcvd := f.pendingData + f.pendingUpdate + f.mu.Unlock() + return fmt.Errorf("received %d-bytes data exceeding the limit %d bytes", rcvd, limit) + } + f.mu.Unlock() + return nil +} + +// onRead is invoked when the application reads the data. It returns the window size +// to be sent to the peer. +func (f *inFlow) onRead(n uint32) uint32 { + f.mu.Lock() + if f.pendingData == 0 { + f.mu.Unlock() + return 0 + } + f.pendingData -= n + if n > f.delta { + n -= f.delta + f.delta = 0 + } else { + f.delta -= n + n = 0 + } + f.pendingUpdate += n + if f.pendingUpdate >= f.limit/4 { + wu := f.pendingUpdate + f.pendingUpdate = 0 + f.mu.Unlock() + return wu + } + f.mu.Unlock() + return 0 +} diff --git a/vendor/google.golang.org/grpc/transport/go16.go b/vendor/google.golang.org/grpc/transport/go16.go index ee1c46bad57..5babcf9b877 100644 --- a/vendor/google.golang.org/grpc/transport/go16.go +++ b/vendor/google.golang.org/grpc/transport/go16.go @@ -1,34 +1,20 @@ // +build go1.6,!go1.7 /* - * Copyright 2016, Google Inc. - * All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Copyright 2016 gRPC authors. * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -36,6 +22,9 @@ package transport import ( "net" + "net/http" + + "google.golang.org/grpc/codes" "golang.org/x/net/context" ) @@ -44,3 +33,19 @@ import ( func dialContext(ctx context.Context, network, address string) (net.Conn, error) { return (&net.Dialer{Cancel: ctx.Done()}).Dial(network, address) } + +// ContextErr converts the error from context package into a StreamError. +func ContextErr(err error) StreamError { + switch err { + case context.DeadlineExceeded: + return streamErrorf(codes.DeadlineExceeded, "%v", err) + case context.Canceled: + return streamErrorf(codes.Canceled, "%v", err) + } + return streamErrorf(codes.Internal, "Unexpected error from context packet: %v", err) +} + +// contextFromRequest returns a background context. +func contextFromRequest(r *http.Request) context.Context { + return context.Background() +} diff --git a/vendor/google.golang.org/grpc/transport/go17.go b/vendor/google.golang.org/grpc/transport/go17.go index 356f13ff197..b7fa6bdb9ca 100644 --- a/vendor/google.golang.org/grpc/transport/go17.go +++ b/vendor/google.golang.org/grpc/transport/go17.go @@ -1,46 +1,52 @@ // +build go1.7 /* - * Copyright 2016, Google Inc. - * All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Copyright 2016 gRPC authors. * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ package transport import ( + "context" "net" + "net/http" + + "google.golang.org/grpc/codes" - "golang.org/x/net/context" + netctx "golang.org/x/net/context" ) // dialContext connects to the address on the named network. func dialContext(ctx context.Context, network, address string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, network, address) } + +// ContextErr converts the error from context package into a StreamError. +func ContextErr(err error) StreamError { + switch err { + case context.DeadlineExceeded, netctx.DeadlineExceeded: + return streamErrorf(codes.DeadlineExceeded, "%v", err) + case context.Canceled, netctx.Canceled: + return streamErrorf(codes.Canceled, "%v", err) + } + return streamErrorf(codes.Internal, "Unexpected error from context packet: %v", err) +} + +// contextFromRequest returns a context from the HTTP Request. +func contextFromRequest(r *http.Request) context.Context { + return r.Context() +} diff --git a/vendor/google.golang.org/grpc/transport/handler_server.go b/vendor/google.golang.org/grpc/transport/handler_server.go index 10b6dc0b19d..f71b7482174 100644 --- a/vendor/google.golang.org/grpc/transport/handler_server.go +++ b/vendor/google.golang.org/grpc/transport/handler_server.go @@ -1,32 +1,18 @@ /* - * Copyright 2016, Google Inc. - * All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Copyright 2016 gRPC authors. * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -47,25 +33,31 @@ import ( "sync" "time" + "github.com/golang/protobuf/proto" "golang.org/x/net/context" "golang.org/x/net/http2" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" ) // NewServerHandlerTransport returns a ServerTransport handling gRPC // from inside an http.Handler. It requires that the http Server // supports HTTP/2. -func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request) (ServerTransport, error) { +func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request, stats stats.Handler) (ServerTransport, error) { if r.ProtoMajor != 2 { return nil, errors.New("gRPC requires HTTP/2") } if r.Method != "POST" { return nil, errors.New("invalid gRPC request method") } - if !validContentType(r.Header.Get("Content-Type")) { + contentType := r.Header.Get("Content-Type") + // TODO: do we assume contentType is lowercase? we did before + contentSubtype, validContentType := contentSubtype(contentType) + if !validContentType { return nil, errors.New("invalid gRPC request content-type") } if _, ok := w.(http.Flusher); !ok { @@ -76,10 +68,13 @@ func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request) (ServerTr } st := &serverHandlerTransport{ - rw: w, - req: r, - closedCh: make(chan struct{}), - writes: make(chan func()), + rw: w, + req: r, + closedCh: make(chan struct{}), + writes: make(chan func()), + contentType: contentType, + contentSubtype: contentSubtype, + stats: stats, } if v := r.Header.Get("grpc-timeout"); v != "" { @@ -91,24 +86,19 @@ func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request) (ServerTr st.timeout = to } - var metakv []string + metakv := []string{"content-type", contentType} if r.Host != "" { metakv = append(metakv, ":authority", r.Host) } for k, vv := range r.Header { k = strings.ToLower(k) - if isReservedHeader(k) && !isWhitelistedPseudoHeader(k) { + if isReservedHeader(k) && !isWhitelistedHeader(k) { continue } for _, v := range vv { - if k == "user-agent" { - // user-agent is special. Copying logic of http_util.go. - if i := strings.LastIndex(v, " "); i == -1 { - // There is no application user agent string being set - continue - } else { - v = v[:i] - } + v, err := decodeMetadataHeader(k, v) + if err != nil { + return nil, streamErrorf(codes.Internal, "malformed binary metadata: %v", err) } metakv = append(metakv, k, v) } @@ -139,6 +129,18 @@ type serverHandlerTransport struct { // ServeHTTP (HandleStreams) goroutine. The channel is closed // when WriteStatus is called. writes chan func() + + // block concurrent WriteStatus calls + // e.g. grpc/(*serverStream).SendMsg/RecvMsg + writeStatusMu sync.Mutex + + // we just mirror the request content-type + contentType string + // we store both contentType and contentSubtype so we don't keep recreating them + // TODO make sure this is consistent across handler_server and http2_server + contentSubtype string + + stats stats.Handler } func (ht *serverHandlerTransport) Close() error { @@ -174,15 +176,24 @@ func (a strAddr) String() string { return string(a) } // do runs fn in the ServeHTTP goroutine. func (ht *serverHandlerTransport) do(fn func()) error { + // Avoid a panic writing to closed channel. Imperfect but maybe good enough. select { - case ht.writes <- fn: - return nil case <-ht.closedCh: return ErrConnClosing + default: + select { + case ht.writes <- fn: + return nil + case <-ht.closedCh: + return ErrConnClosing + } } } -func (ht *serverHandlerTransport) WriteStatus(s *Stream, statusCode codes.Code, statusDesc string) error { +func (ht *serverHandlerTransport) WriteStatus(s *Stream, st *status.Status) error { + ht.writeStatusMu.Lock() + defer ht.writeStatusMu.Unlock() + err := ht.do(func() { ht.writeCommonHeaders(s) @@ -192,10 +203,21 @@ func (ht *serverHandlerTransport) WriteStatus(s *Stream, statusCode codes.Code, ht.rw.(http.Flusher).Flush() h := ht.rw.Header() - h.Set("Grpc-Status", fmt.Sprintf("%d", statusCode)) - if statusDesc != "" { - h.Set("Grpc-Message", encodeGrpcMessage(statusDesc)) + h.Set("Grpc-Status", fmt.Sprintf("%d", st.Code())) + if m := st.Message(); m != "" { + h.Set("Grpc-Message", encodeGrpcMessage(m)) + } + + if p := st.Proto(); p != nil && len(p.Details) > 0 { + stBytes, err := proto.Marshal(p) + if err != nil { + // TODO: return error instead, when callers are able to handle it. + panic(err) + } + + h.Set("Grpc-Status-Details-Bin", encodeBinHeader(stBytes)) } + if md := s.Trailer(); len(md) > 0 { for k, vv := range md { // Clients don't tolerate reading restricted headers after some non restricted ones were sent. @@ -203,15 +225,21 @@ func (ht *serverHandlerTransport) WriteStatus(s *Stream, statusCode codes.Code, continue } for _, v := range vv { - // http2 ResponseWriter mechanism to - // send undeclared Trailers after the - // headers have possibly been written. - h.Add(http2.TrailerPrefix+k, v) + // http2 ResponseWriter mechanism to send undeclared Trailers after + // the headers have possibly been written. + h.Add(http2.TrailerPrefix+k, encodeMetadataHeader(k, v)) } } } }) - close(ht.writes) + + if err == nil { // transport has not been closed + if ht.stats != nil { + ht.stats.HandleRPC(s.Context(), &stats.OutTrailer{}) + } + ht.Close() + close(ht.writes) + } return err } @@ -225,7 +253,7 @@ func (ht *serverHandlerTransport) writeCommonHeaders(s *Stream) { h := ht.rw.Header() h["Date"] = nil // suppress Date to make tests happy; TODO: restore - h.Set("Content-Type", "application/grpc") + h.Set("Content-Type", ht.contentType) // Predeclare trailers we'll set later in WriteStatus (after the body). // This is a SHOULD in the HTTP RFC, and the way you add (known) @@ -234,15 +262,17 @@ func (ht *serverHandlerTransport) writeCommonHeaders(s *Stream) { // and https://golang.org/pkg/net/http/#example_ResponseWriter_trailers h.Add("Trailer", "Grpc-Status") h.Add("Trailer", "Grpc-Message") + h.Add("Trailer", "Grpc-Status-Details-Bin") if s.sendCompress != "" { h.Set("Grpc-Encoding", s.sendCompress) } } -func (ht *serverHandlerTransport) Write(s *Stream, data []byte, opts *Options) error { +func (ht *serverHandlerTransport) Write(s *Stream, hdr []byte, data []byte, opts *Options) error { return ht.do(func() { ht.writeCommonHeaders(s) + ht.rw.Write(hdr) ht.rw.Write(data) if !opts.Delay { ht.rw.(http.Flusher).Flush() @@ -251,7 +281,7 @@ func (ht *serverHandlerTransport) Write(s *Stream, data []byte, opts *Options) e } func (ht *serverHandlerTransport) WriteHeader(s *Stream, md metadata.MD) error { - return ht.do(func() { + err := ht.do(func() { ht.writeCommonHeaders(s) h := ht.rw.Header() for k, vv := range md { @@ -260,23 +290,31 @@ func (ht *serverHandlerTransport) WriteHeader(s *Stream, md metadata.MD) error { continue } for _, v := range vv { + v = encodeMetadataHeader(k, v) h.Add(k, v) } } ht.rw.WriteHeader(200) ht.rw.(http.Flusher).Flush() }) + + if err == nil { + if ht.stats != nil { + ht.stats.HandleRPC(s.Context(), &stats.OutHeader{}) + } + } + return err } func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), traceCtx func(context.Context, string) context.Context) { // With this transport type there will be exactly 1 stream: this HTTP request. - var ctx context.Context + ctx := contextFromRequest(ht.req) var cancel context.CancelFunc if ht.timeoutSet { - ctx, cancel = context.WithTimeout(context.Background(), ht.timeout) + ctx, cancel = context.WithTimeout(ctx, ht.timeout) } else { - ctx, cancel = context.WithCancel(context.Background()) + ctx, cancel = context.WithCancel(ctx) } // requestOver is closed when either the request's context is done @@ -300,13 +338,14 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace req := ht.req s := &Stream{ - id: 0, // irrelevant - windowHandler: func(int) {}, // nothing - cancel: cancel, - buf: newRecvBuffer(), - st: ht, - method: req.URL.Path, - recvCompress: req.Header.Get("grpc-encoding"), + id: 0, // irrelevant + requestRead: func(int) {}, + cancel: cancel, + buf: newRecvBuffer(), + st: ht, + method: req.URL.Path, + recvCompress: req.Header.Get("grpc-encoding"), + contentSubtype: ht.contentSubtype, } pr := &peer.Peer{ Addr: ht.RemoteAddr(), @@ -314,10 +353,21 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace if req.TLS != nil { pr.AuthInfo = credentials.TLSInfo{State: *req.TLS} } - ctx = metadata.NewContext(ctx, ht.headerMD) - ctx = peer.NewContext(ctx, pr) - s.ctx = newContextWithStream(ctx, s) - s.dec = &recvBufferReader{ctx: s.ctx, recv: s.buf} + ctx = metadata.NewIncomingContext(ctx, ht.headerMD) + s.ctx = peer.NewContext(ctx, pr) + if ht.stats != nil { + s.ctx = ht.stats.TagRPC(s.ctx, &stats.RPCTagInfo{FullMethodName: s.method}) + inHeader := &stats.InHeader{ + FullMethod: s.method, + RemoteAddr: ht.RemoteAddr(), + Compression: s.recvCompress, + } + ht.stats.HandleRPC(s.ctx, inHeader) + } + s.trReader = &transportReader{ + reader: &recvBufferReader{ctx: s.ctx, ctxDone: s.ctx.Done(), recv: s.buf}, + windowHandler: func(int) {}, + } // readerDone is closed when the Body.Read-ing goroutine exits. readerDone := make(chan struct{}) @@ -329,11 +379,11 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace for buf := make([]byte, readSize); ; { n, err := req.Body.Read(buf) if n > 0 { - s.buf.put(&recvMsg{data: buf[:n:n]}) + s.buf.put(recvMsg{data: buf[:n:n]}) buf = buf[n:] } if err != nil { - s.buf.put(&recvMsg{err: mapRecvMsgError(err)}) + s.buf.put(recvMsg{err: mapRecvMsgError(err)}) return } if len(buf) == 0 { @@ -370,6 +420,10 @@ func (ht *serverHandlerTransport) runStream() { } } +func (ht *serverHandlerTransport) IncrMsgSent() {} + +func (ht *serverHandlerTransport) IncrMsgRecv() {} + func (ht *serverHandlerTransport) Drain() { panic("Drain() is not implemented") } diff --git a/vendor/google.golang.org/grpc/transport/http2_client.go b/vendor/google.golang.org/grpc/transport/http2_client.go index 3936b9cf9fa..1fdabd954ef 100644 --- a/vendor/google.golang.org/grpc/transport/http2_client.go +++ b/vendor/google.golang.org/grpc/transport/http2_client.go @@ -1,115 +1,124 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ package transport import ( - "bytes" - "fmt" "io" "math" "net" "strings" "sync" + "sync/atomic" "time" "golang.org/x/net/context" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" + + "google.golang.org/grpc/channelz" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" - "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" ) // http2Client implements the ClientTransport interface with HTTP2. type http2Client struct { ctx context.Context - target string // server name/addr + cancel context.CancelFunc + ctxDone <-chan struct{} // Cache the ctx.Done() chan. userAgent string md interface{} conn net.Conn // underlying communication channel + loopy *loopyWriter remoteAddr net.Addr localAddr net.Addr authInfo credentials.AuthInfo // auth info about the connection - nextID uint32 // the next stream ID to be used - // writableChan synchronizes write access to the transport. - // A writer acquires the write lock by sending a value on writableChan - // and releases it by receiving from writableChan. - writableChan chan int - // shutdownChan is closed when Close is called. - // Blocking operations should select on shutdownChan to avoid - // blocking forever after Close. - // TODO(zhaoq): Maybe have a channel context? - shutdownChan chan struct{} - // errorChan is closed to notify the I/O error to the caller. - errorChan chan struct{} + readerDone chan struct{} // sync point to enable testing. + writerDone chan struct{} // sync point to enable testing. // goAway is closed to notify the upper layer (i.e., addrConn.transportMonitor) // that the server sent GoAway on this transport. goAway chan struct{} + // awakenKeepalive is used to wake up keepalive when after it has gone dormant. + awakenKeepalive chan struct{} framer *framer - hBuf *bytes.Buffer // the buffer for HPACK encoding - hEnc *hpack.Encoder // HPACK encoder - // controlBuf delivers all the control related tasks (e.g., window // updates, reset streams, and various settings) to the controller. - controlBuf *recvBuffer - fc *inFlow - // sendQuotaPool provides flow control to outbound message. - sendQuotaPool *quotaPool - // streamsQuota limits the max number of concurrent streams. - streamsQuota *quotaPool - + controlBuf *controlBuffer + fc *trInFlow // The scheme used: https if TLS is on, http otherwise. scheme string + isSecure bool + creds []credentials.PerRPCCredentials - mu sync.Mutex // guard the following variables - state transportState // the state of underlying connection + // Boolean to keep track of reading activity on transport. + // 1 is true and 0 is false. + activity uint32 // Accessed atomically. + kp keepalive.ClientParameters + + statsHandler stats.Handler + + initialWindowSize int32 + + bdpEst *bdpEstimator + // onSuccess is a callback that client transport calls upon + // receiving server preface to signal that a succefull HTTP2 + // connection was established. + onSuccess func() + + maxConcurrentStreams uint32 + streamQuota int64 + streamsQuotaAvailable chan struct{} + waitingStreams uint32 + nextID uint32 + + mu sync.Mutex // guard the following variables + state transportState activeStreams map[uint32]*Stream - // The max number of concurrent streams - maxStreams int - // the per-stream outbound flow control window size set by the peer. - streamSendQuota uint32 - // goAwayID records the Last-Stream-ID in the GoAway frame from the server. - goAwayID uint32 // prevGoAway ID records the Last-Stream-ID in the previous GOAway frame. prevGoAwayID uint32 + // goAwayReason records the http2.ErrCode and debug data received with the + // GoAway frame. + goAwayReason GoAwayReason + + // Fields below are for channelz metric collection. + channelzID int64 // channelz unique identification number + czmu sync.RWMutex + kpCount int64 + // The number of streams that have started, including already finished ones. + streamsStarted int64 + // The number of streams that have ended successfully by receiving EoS bit set + // frame from server. + streamsSucceeded int64 + streamsFailed int64 + lastStreamCreated time.Time + msgSent int64 + msgRecv int64 + lastMsgSent time.Time + lastMsgRecv time.Time } func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error), addr string) (net.Conn, error) { @@ -120,18 +129,6 @@ func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error } func isTemporary(err error) bool { - switch err { - case io.EOF: - // Connection closures may be resolved upon retry, and are thus - // treated as temporary. - return true - case context.DeadlineExceeded: - // In Go 1.7, context.DeadlineExceeded implements Timeout(), and this - // special case is not needed. Until then, we need to keep this - // clause. - return true - } - switch err := err.(type) { case interface { Temporary() bool @@ -144,20 +141,27 @@ func isTemporary(err error) bool { // temporary. return err.Timeout() } - return false + return true } // newHTTP2Client constructs a connected ClientTransport to addr based on HTTP2 // and starts to receive messages on it. Non-nil error returns if construction // fails. -func newHTTP2Client(ctx context.Context, addr TargetInfo, opts ConnectOptions) (_ ClientTransport, err error) { +func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts ConnectOptions, onSuccess func()) (_ ClientTransport, err error) { scheme := "http" - conn, err := dial(ctx, opts.Dialer, addr.Addr) + ctx, cancel := context.WithCancel(ctx) + defer func() { + if err != nil { + cancel() + } + }() + + conn, err := dial(connectCtx, opts.Dialer, addr.Addr) if err != nil { if opts.FailOnNonTempDialError { - return nil, connectionErrorf(isTemporary(err), err, "transport: %v", err) + return nil, connectionErrorf(isTemporary(err), err, "transport: error while dialing: %v", err) } - return nil, connectionErrorf(true, err, "transport: %v", err) + return nil, connectionErrorf(true, err, "transport: Error while dialing %v", err) } // Any further errors will close the underlying connection defer func(conn net.Conn) { @@ -165,49 +169,95 @@ func newHTTP2Client(ctx context.Context, addr TargetInfo, opts ConnectOptions) ( conn.Close() } }(conn) - var authInfo credentials.AuthInfo + var ( + isSecure bool + authInfo credentials.AuthInfo + ) if creds := opts.TransportCredentials; creds != nil { scheme = "https" - conn, authInfo, err = creds.ClientHandshake(ctx, addr.Addr, conn) + conn, authInfo, err = creds.ClientHandshake(connectCtx, addr.Authority, conn) if err != nil { - // Credentials handshake errors are typically considered permanent - // to avoid retrying on e.g. bad certificates. - temp := isTemporary(err) - return nil, connectionErrorf(temp, err, "transport: %v", err) + return nil, connectionErrorf(isTemporary(err), err, "transport: authentication handshake failed: %v", err) } + isSecure = true + } + kp := opts.KeepaliveParams + // Validate keepalive parameters. + if kp.Time == 0 { + kp.Time = defaultClientKeepaliveTime + } + if kp.Timeout == 0 { + kp.Timeout = defaultClientKeepaliveTimeout } - ua := primaryUA - if opts.UserAgent != "" { - ua = opts.UserAgent + " " + ua + dynamicWindow := true + icwz := int32(initialWindowSize) + if opts.InitialConnWindowSize >= defaultWindowSize { + icwz = opts.InitialConnWindowSize + dynamicWindow = false + } + writeBufSize := defaultWriteBufSize + if opts.WriteBufferSize > 0 { + writeBufSize = opts.WriteBufferSize + } + readBufSize := defaultReadBufSize + if opts.ReadBufferSize > 0 { + readBufSize = opts.ReadBufferSize } - var buf bytes.Buffer t := &http2Client{ - ctx: ctx, - target: addr.Addr, - userAgent: ua, - md: addr.Metadata, - conn: conn, - remoteAddr: conn.RemoteAddr(), - localAddr: conn.LocalAddr(), - authInfo: authInfo, - // The client initiated stream id is odd starting from 1. - nextID: 1, - writableChan: make(chan int, 1), - shutdownChan: make(chan struct{}), - errorChan: make(chan struct{}), - goAway: make(chan struct{}), - framer: newFramer(conn), - hBuf: &buf, - hEnc: hpack.NewEncoder(&buf), - controlBuf: newRecvBuffer(), - fc: &inFlow{limit: initialConnWindowSize}, - sendQuotaPool: newQuotaPool(defaultWindowSize), - scheme: scheme, - state: reachable, - activeStreams: make(map[uint32]*Stream), - creds: opts.PerRPCCredentials, - maxStreams: math.MaxInt32, - streamSendQuota: defaultWindowSize, + ctx: ctx, + ctxDone: ctx.Done(), // Cache Done chan. + cancel: cancel, + userAgent: opts.UserAgent, + md: addr.Metadata, + conn: conn, + remoteAddr: conn.RemoteAddr(), + localAddr: conn.LocalAddr(), + authInfo: authInfo, + readerDone: make(chan struct{}), + writerDone: make(chan struct{}), + goAway: make(chan struct{}), + awakenKeepalive: make(chan struct{}, 1), + framer: newFramer(conn, writeBufSize, readBufSize), + fc: &trInFlow{limit: uint32(icwz)}, + scheme: scheme, + activeStreams: make(map[uint32]*Stream), + isSecure: isSecure, + creds: opts.PerRPCCredentials, + kp: kp, + statsHandler: opts.StatsHandler, + initialWindowSize: initialWindowSize, + onSuccess: onSuccess, + nextID: 1, + maxConcurrentStreams: defaultMaxStreamsClient, + streamQuota: defaultMaxStreamsClient, + streamsQuotaAvailable: make(chan struct{}, 1), + } + t.controlBuf = newControlBuffer(t.ctxDone) + if opts.InitialWindowSize >= defaultWindowSize { + t.initialWindowSize = opts.InitialWindowSize + dynamicWindow = false + } + if dynamicWindow { + t.bdpEst = &bdpEstimator{ + bdp: initialWindowSize, + updateFlowControl: t.updateFlowControl, + } + } + // Make sure awakenKeepalive can't be written upon. + // keepalive routine will make it writable, if need be. + t.awakenKeepalive <- struct{}{} + if t.statsHandler != nil { + t.ctx = t.statsHandler.TagConn(t.ctx, &stats.ConnTagInfo{ + RemoteAddr: t.remoteAddr, + LocalAddr: t.localAddr, + }) + connBegin := &stats.ConnBegin{ + Client: true, + } + t.statsHandler.HandleConn(t.ctx, connBegin) + } + if channelz.IsOn() { + t.channelzID = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, "") } // Start the reader goroutine for incoming message. Each transport has // a dedicated goroutine which reads HTTP2 frame from network. Then it @@ -217,42 +267,40 @@ func newHTTP2Client(ctx context.Context, addr TargetInfo, opts ConnectOptions) ( n, err := t.conn.Write(clientPreface) if err != nil { t.Close() - return nil, connectionErrorf(true, err, "transport: %v", err) + return nil, connectionErrorf(true, err, "transport: failed to write client preface: %v", err) } if n != len(clientPreface) { t.Close() return nil, connectionErrorf(true, err, "transport: preface mismatch, wrote %d bytes; want %d", n, len(clientPreface)) } - if initialWindowSize != defaultWindowSize { - err = t.framer.writeSettings(true, http2.Setting{ + if t.initialWindowSize != defaultWindowSize { + err = t.framer.fr.WriteSettings(http2.Setting{ ID: http2.SettingInitialWindowSize, - Val: uint32(initialWindowSize), + Val: uint32(t.initialWindowSize), }) } else { - err = t.framer.writeSettings(true) + err = t.framer.fr.WriteSettings() } if err != nil { t.Close() - return nil, connectionErrorf(true, err, "transport: %v", err) + return nil, connectionErrorf(true, err, "transport: failed to write initial settings frame: %v", err) } // Adjust the connection flow control window if needed. - if delta := uint32(initialConnWindowSize - defaultWindowSize); delta > 0 { - if err := t.framer.writeWindowUpdate(true, 0, delta); err != nil { + if delta := uint32(icwz - defaultWindowSize); delta > 0 { + if err := t.framer.fr.WriteWindowUpdate(0, delta); err != nil { t.Close() - return nil, connectionErrorf(true, err, "transport: %v", err) + return nil, connectionErrorf(true, err, "transport: failed to write window update: %v", err) } } - go t.controller() - t.writableChan <- 0 - if stats.On() { - t.ctx = stats.TagConn(t.ctx, &stats.ConnTagInfo{ - RemoteAddr: t.remoteAddr, - LocalAddr: t.localAddr, - }) - connBegin := &stats.ConnBegin{ - Client: true, - } - stats.HandleConn(t.ctx, connBegin) + t.framer.writer.Flush() + go func() { + t.loopy = newLoopyWriter(clientSide, t.framer, t.controlBuf, t.bdpEst) + t.loopy.run() + t.conn.Close() + close(t.writerDone) + }() + if t.kp.Time != infinity { + go t.keepalive() } return t, nil } @@ -260,35 +308,35 @@ func newHTTP2Client(ctx context.Context, addr TargetInfo, opts ConnectOptions) ( func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream { // TODO(zhaoq): Handle uint32 overflow of Stream.id. s := &Stream{ - id: t.nextID, - done: make(chan struct{}), - goAway: make(chan struct{}), - method: callHdr.Method, - sendCompress: callHdr.SendCompress, - buf: newRecvBuffer(), - fc: &inFlow{limit: initialWindowSize}, - sendQuotaPool: newQuotaPool(int(t.streamSendQuota)), - headerChan: make(chan struct{}), - } - t.nextID += 2 - s.windowHandler = func(n int) { - t.updateWindow(s, uint32(n)) + done: make(chan struct{}), + method: callHdr.Method, + sendCompress: callHdr.SendCompress, + buf: newRecvBuffer(), + headerChan: make(chan struct{}), + contentSubtype: callHdr.ContentSubtype, + } + s.wq = newWriteQuota(defaultWriteQuota, s.done) + s.requestRead = func(n int) { + t.adjustWindow(s, uint32(n)) } // The client side stream context should have exactly the same life cycle with the user provided context. // That means, s.ctx should be read-only. And s.ctx is done iff ctx is done. // So we use the original context here instead of creating a copy. s.ctx = ctx - s.dec = &recvBufferReader{ - ctx: s.ctx, - goAway: s.goAway, - recv: s.buf, + s.trReader = &transportReader{ + reader: &recvBufferReader{ + ctx: s.ctx, + ctxDone: s.ctx.Done(), + recv: s.buf, + }, + windowHandler: func(n int) { + t.updateWindow(s, uint32(n)) + }, } return s } -// NewStream creates a stream and register it into the transport as "active" -// streams. -func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Stream, err error) { +func (t *http2Client) getPeer() *peer.Peer { pr := &peer.Peer{ Addr: t.remoteAddr, } @@ -296,310 +344,388 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea if t.authInfo != nil { pr.AuthInfo = t.authInfo } - userCtx := ctx - ctx = peer.NewContext(ctx, pr) - authData := make(map[string]string) - for _, c := range t.creds { - // Construct URI required to get auth request metadata. - var port string - if pos := strings.LastIndex(t.target, ":"); pos != -1 { - // Omit port if it is the default one. - if t.target[pos+1:] != "443" { - port = ":" + t.target[pos+1:] - } - } - pos := strings.LastIndex(callHdr.Method, "/") - if pos == -1 { - return nil, streamErrorf(codes.InvalidArgument, "transport: malformed method name: %q", callHdr.Method) - } - audience := "https://" + callHdr.Host + port + callHdr.Method[:pos] - data, err := c.GetRequestMetadata(ctx, audience) - if err != nil { - return nil, streamErrorf(codes.InvalidArgument, "transport: %v", err) - } - for k, v := range data { - authData[k] = v - } - } - t.mu.Lock() - if t.activeStreams == nil { - t.mu.Unlock() - return nil, ErrConnClosing - } - if t.state == draining { - t.mu.Unlock() - return nil, ErrStreamDrain - } - if t.state != reachable { - t.mu.Unlock() - return nil, ErrConnClosing - } - checkStreamsQuota := t.streamsQuota != nil - t.mu.Unlock() - if checkStreamsQuota { - sq, err := wait(ctx, nil, nil, t.shutdownChan, t.streamsQuota.acquire()) - if err != nil { - return nil, err - } - // Returns the quota balance back. - if sq > 1 { - t.streamsQuota.add(sq - 1) - } - } - if _, err := wait(ctx, nil, nil, t.shutdownChan, t.writableChan); err != nil { - // Return the quota back now because there is no stream returned to the caller. - if _, ok := err.(StreamError); ok && checkStreamsQuota { - t.streamsQuota.add(1) - } - return nil, err - } - t.mu.Lock() - if t.state == draining { - t.mu.Unlock() - if checkStreamsQuota { - t.streamsQuota.add(1) - } - // Need to make t writable again so that the rpc in flight can still proceed. - t.writableChan <- 0 - return nil, ErrStreamDrain - } - if t.state != reachable { - t.mu.Unlock() - return nil, ErrConnClosing - } - s := t.newStream(ctx, callHdr) - s.clientStatsCtx = userCtx - t.activeStreams[s.id] = s + return pr +} - // This stream is not counted when applySetings(...) initialize t.streamsQuota. - // Reset t.streamsQuota to the right value. - var reset bool - if !checkStreamsQuota && t.streamsQuota != nil { - reset = true +func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) ([]hpack.HeaderField, error) { + aud := t.createAudience(callHdr) + authData, err := t.getTrAuthData(ctx, aud) + if err != nil { + return nil, err } - t.mu.Unlock() - if reset { - t.streamsQuota.add(-1) + callAuthData, err := t.getCallAuthData(ctx, aud, callHdr) + if err != nil { + return nil, err } - - // HPACK encodes various headers. Note that once WriteField(...) is - // called, the corresponding headers/continuation frame has to be sent - // because hpack.Encoder is stateful. - t.hBuf.Reset() - t.hEnc.WriteField(hpack.HeaderField{Name: ":method", Value: "POST"}) - t.hEnc.WriteField(hpack.HeaderField{Name: ":scheme", Value: t.scheme}) - t.hEnc.WriteField(hpack.HeaderField{Name: ":path", Value: callHdr.Method}) - t.hEnc.WriteField(hpack.HeaderField{Name: ":authority", Value: callHdr.Host}) - t.hEnc.WriteField(hpack.HeaderField{Name: "content-type", Value: "application/grpc"}) - t.hEnc.WriteField(hpack.HeaderField{Name: "user-agent", Value: t.userAgent}) - t.hEnc.WriteField(hpack.HeaderField{Name: "te", Value: "trailers"}) + // TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields + // first and create a slice of that exact size. + // Make the slice of certain predictable size to reduce allocations made by append. + hfLen := 7 // :method, :scheme, :path, :authority, content-type, user-agent, te + hfLen += len(authData) + len(callAuthData) + headerFields := make([]hpack.HeaderField, 0, hfLen) + headerFields = append(headerFields, hpack.HeaderField{Name: ":method", Value: "POST"}) + headerFields = append(headerFields, hpack.HeaderField{Name: ":scheme", Value: t.scheme}) + headerFields = append(headerFields, hpack.HeaderField{Name: ":path", Value: callHdr.Method}) + headerFields = append(headerFields, hpack.HeaderField{Name: ":authority", Value: callHdr.Host}) + headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: contentType(callHdr.ContentSubtype)}) + headerFields = append(headerFields, hpack.HeaderField{Name: "user-agent", Value: t.userAgent}) + headerFields = append(headerFields, hpack.HeaderField{Name: "te", Value: "trailers"}) if callHdr.SendCompress != "" { - t.hEnc.WriteField(hpack.HeaderField{Name: "grpc-encoding", Value: callHdr.SendCompress}) + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-encoding", Value: callHdr.SendCompress}) } if dl, ok := ctx.Deadline(); ok { // Send out timeout regardless its value. The server can detect timeout context by itself. + // TODO(mmukhi): Perhaps this field should be updated when actually writing out to the wire. timeout := dl.Sub(time.Now()) - t.hEnc.WriteField(hpack.HeaderField{Name: "grpc-timeout", Value: encodeTimeout(timeout)}) + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-timeout", Value: encodeTimeout(timeout)}) } - for k, v := range authData { - // Capital header names are illegal in HTTP/2. - k = strings.ToLower(k) - t.hEnc.WriteField(hpack.HeaderField{Name: k, Value: v}) + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) } - var ( - hasMD bool - endHeaders bool - ) - if md, ok := metadata.FromContext(ctx); ok { - hasMD = true - for k, v := range md { + for k, v := range callAuthData { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) + } + if b := stats.OutgoingTags(ctx); b != nil { + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-tags-bin", Value: encodeBinHeader(b)}) + } + if b := stats.OutgoingTrace(ctx); b != nil { + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-trace-bin", Value: encodeBinHeader(b)}) + } + + if md, added, ok := metadata.FromOutgoingContextRaw(ctx); ok { + var k string + for _, vv := range added { + for i, v := range vv { + if i%2 == 0 { + k = v + continue + } + // HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set. + if isReservedHeader(k) { + continue + } + headerFields = append(headerFields, hpack.HeaderField{Name: strings.ToLower(k), Value: encodeMetadataHeader(k, v)}) + } + } + for k, vv := range md { // HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set. if isReservedHeader(k) { continue } - for _, entry := range v { - t.hEnc.WriteField(hpack.HeaderField{Name: k, Value: entry}) + for _, v := range vv { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) } } } if md, ok := t.md.(*metadata.MD); ok { - for k, v := range *md { + for k, vv := range *md { if isReservedHeader(k) { continue } - for _, entry := range v { - t.hEnc.WriteField(hpack.HeaderField{Name: k, Value: entry}) + for _, v := range vv { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) } } } - first := true - bufLen := t.hBuf.Len() - // Sends the headers in a single batch even when they span multiple frames. - for !endHeaders { - size := t.hBuf.Len() - if size > http2MaxFrameLen { - size = http2MaxFrameLen - } else { - endHeaders = true + return headerFields, nil +} + +func (t *http2Client) createAudience(callHdr *CallHdr) string { + // Create an audience string only if needed. + if len(t.creds) == 0 && callHdr.Creds == nil { + return "" + } + // Construct URI required to get auth request metadata. + // Omit port if it is the default one. + host := strings.TrimSuffix(callHdr.Host, ":443") + pos := strings.LastIndex(callHdr.Method, "/") + if pos == -1 { + pos = len(callHdr.Method) + } + return "https://" + host + callHdr.Method[:pos] +} + +func (t *http2Client) getTrAuthData(ctx context.Context, audience string) (map[string]string, error) { + authData := map[string]string{} + for _, c := range t.creds { + data, err := c.GetRequestMetadata(ctx, audience) + if err != nil { + if _, ok := status.FromError(err); ok { + return nil, err + } + + return nil, streamErrorf(codes.Unauthenticated, "transport: %v", err) } - var flush bool - if endHeaders && (hasMD || callHdr.Flush) { - flush = true + for k, v := range data { + // Capital header names are illegal in HTTP/2. + k = strings.ToLower(k) + authData[k] = v } - if first { - // Sends a HeadersFrame to server to start a new stream. - p := http2.HeadersFrameParam{ - StreamID: s.id, - BlockFragment: t.hBuf.Next(size), - EndStream: false, - EndHeaders: endHeaders, + } + return authData, nil +} + +func (t *http2Client) getCallAuthData(ctx context.Context, audience string, callHdr *CallHdr) (map[string]string, error) { + callAuthData := map[string]string{} + // Check if credentials.PerRPCCredentials were provided via call options. + // Note: if these credentials are provided both via dial options and call + // options, then both sets of credentials will be applied. + if callCreds := callHdr.Creds; callCreds != nil { + if !t.isSecure && callCreds.RequireTransportSecurity() { + return nil, streamErrorf(codes.Unauthenticated, "transport: cannot send secure credentials on an insecure connection") + } + data, err := callCreds.GetRequestMetadata(ctx, audience) + if err != nil { + return nil, streamErrorf(codes.Internal, "transport: %v", err) + } + for k, v := range data { + // Capital header names are illegal in HTTP/2 + k = strings.ToLower(k) + callAuthData[k] = v + } + } + return callAuthData, nil +} + +// NewStream creates a stream and registers it into the transport as "active" +// streams. +func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Stream, err error) { + ctx = peer.NewContext(ctx, t.getPeer()) + headerFields, err := t.createHeaderFields(ctx, callHdr) + if err != nil { + return nil, err + } + s := t.newStream(ctx, callHdr) + cleanup := func(err error) { + if s.swapState(streamDone) == streamDone { + // If it was already done, return. + return + } + // The stream was unprocessed by the server. + atomic.StoreUint32(&s.unprocessed, 1) + s.write(recvMsg{err: err}) + close(s.done) + // If headerChan isn't closed, then close it. + if atomic.SwapUint32(&s.headerDone, 1) == 0 { + close(s.headerChan) + } + + } + hdr := &headerFrame{ + hf: headerFields, + endStream: false, + initStream: func(id uint32) (bool, error) { + t.mu.Lock() + if state := t.state; state != reachable { + t.mu.Unlock() + // Do a quick cleanup. + err := error(errStreamDrain) + if state == closing { + err = ErrConnClosing + } + cleanup(err) + return false, err } - // Do a force flush for the buffered frames iff it is the last headers frame - // and there is header metadata to be sent. Otherwise, there is flushing until - // the corresponding data frame is written. - err = t.framer.writeHeaders(flush, p) - first = false - } else { - // Sends Continuation frames for the leftover headers. - err = t.framer.writeContinuation(flush, s.id, endHeaders, t.hBuf.Next(size)) + t.activeStreams[id] = s + if channelz.IsOn() { + t.czmu.Lock() + t.streamsStarted++ + t.lastStreamCreated = time.Now() + t.czmu.Unlock() + } + var sendPing bool + // If the number of active streams change from 0 to 1, then check if keepalive + // has gone dormant. If so, wake it up. + if len(t.activeStreams) == 1 { + select { + case t.awakenKeepalive <- struct{}{}: + sendPing = true + // Fill the awakenKeepalive channel again as this channel must be + // kept non-writable except at the point that the keepalive() + // goroutine is waiting either to be awaken or shutdown. + t.awakenKeepalive <- struct{}{} + default: + } + } + t.mu.Unlock() + return sendPing, nil + }, + onOrphaned: cleanup, + wq: s.wq, + } + firstTry := true + var ch chan struct{} + checkForStreamQuota := func(it interface{}) bool { + if t.streamQuota <= 0 { // Can go negative if server decreases it. + if firstTry { + t.waitingStreams++ + } + ch = t.streamsQuotaAvailable + return false } + if !firstTry { + t.waitingStreams-- + } + t.streamQuota-- + h := it.(*headerFrame) + h.streamID = t.nextID + t.nextID += 2 + s.id = h.streamID + s.fc = &inFlow{limit: uint32(t.initialWindowSize)} + if t.streamQuota > 0 && t.waitingStreams > 0 { + select { + case t.streamsQuotaAvailable <- struct{}{}: + default: + } + } + return true + } + for { + success, err := t.controlBuf.executeAndPut(checkForStreamQuota, hdr) if err != nil { - t.notifyError(err) - return nil, connectionErrorf(true, err, "transport: %v", err) + return nil, err + } + if success { + break + } + firstTry = false + select { + case <-ch: + case <-s.ctx.Done(): + return nil, ContextErr(s.ctx.Err()) + case <-t.goAway: + return nil, errStreamDrain + case <-t.ctx.Done(): + return nil, ErrConnClosing } } - if stats.On() { + if t.statsHandler != nil { outHeader := &stats.OutHeader{ Client: true, - WireLength: bufLen, FullMethod: callHdr.Method, RemoteAddr: t.remoteAddr, LocalAddr: t.localAddr, Compression: callHdr.SendCompress, } - stats.HandleRPC(s.clientStatsCtx, outHeader) + t.statsHandler.HandleRPC(s.ctx, outHeader) } - t.writableChan <- 0 return s, nil } // CloseStream clears the footprint of a stream when the stream is not needed any more. // This must not be executed in reader's goroutine. func (t *http2Client) CloseStream(s *Stream, err error) { - var updateStreams bool - t.mu.Lock() - if t.activeStreams == nil { - t.mu.Unlock() - return - } - if t.streamsQuota != nil { - updateStreams = true + var ( + rst bool + rstCode http2.ErrCode + ) + if err != nil { + rst = true + rstCode = http2.ErrCodeCancel } - delete(t.activeStreams, s.id) - if t.state == draining && len(t.activeStreams) == 0 { - // The transport is draining and s is the last live stream on t. - t.mu.Unlock() - t.Close() + t.closeStream(s, err, rst, rstCode, nil, nil, false) +} + +func (t *http2Client) closeStream(s *Stream, err error, rst bool, rstCode http2.ErrCode, st *status.Status, mdata map[string][]string, eosReceived bool) { + // Set stream status to done. + if s.swapState(streamDone) == streamDone { + // If it was already done, return. return } - t.mu.Unlock() - if updateStreams { - t.streamsQuota.add(1) - } - s.mu.Lock() - if q := s.fc.resetPendingData(); q > 0 { - if n := t.fc.onRead(q); n > 0 { - t.controlBuf.put(&windowUpdate{0, n}) - } + // status and trailers can be updated here without any synchronization because the stream goroutine will + // only read it after it sees an io.EOF error from read or write and we'll write those errors + // only after updating this. + s.status = st + if len(mdata) > 0 { + s.trailer = mdata } - if s.state == streamDone { - s.mu.Unlock() - return + if err != nil { + // This will unblock reads eventually. + s.write(recvMsg{err: err}) } - if !s.headerDone { + // This will unblock write. + close(s.done) + // If headerChan isn't closed, then close it. + if atomic.SwapUint32(&s.headerDone, 1) == 0 { close(s.headerChan) - s.headerDone = true } - s.state = streamDone - s.mu.Unlock() - if se, ok := err.(StreamError); ok && se.Code != codes.DeadlineExceeded { - t.controlBuf.put(&resetStream{s.id, http2.ErrCodeCancel}) + cleanup := &cleanupStream{ + streamID: s.id, + onWrite: func() { + t.mu.Lock() + if t.activeStreams != nil { + delete(t.activeStreams, s.id) + } + t.mu.Unlock() + if channelz.IsOn() { + t.czmu.Lock() + if eosReceived { + t.streamsSucceeded++ + } else { + t.streamsFailed++ + } + t.czmu.Unlock() + } + }, + rst: rst, + rstCode: rstCode, + } + addBackStreamQuota := func(interface{}) bool { + t.streamQuota++ + if t.streamQuota > 0 && t.waitingStreams > 0 { + select { + case t.streamsQuotaAvailable <- struct{}{}: + default: + } + } + return true } + t.controlBuf.executeAndPut(addBackStreamQuota, cleanup) } // Close kicks off the shutdown process of the transport. This should be called // only once on a transport. Once it is called, the transport should not be // accessed any more. -func (t *http2Client) Close() (err error) { +func (t *http2Client) Close() error { t.mu.Lock() + // Make sure we only Close once. if t.state == closing { t.mu.Unlock() - return - } - if t.state == reachable || t.state == draining { - close(t.errorChan) + return nil } t.state = closing - t.mu.Unlock() - close(t.shutdownChan) - err = t.conn.Close() - t.mu.Lock() streams := t.activeStreams t.activeStreams = nil t.mu.Unlock() + t.controlBuf.finish() + t.cancel() + err := t.conn.Close() + if channelz.IsOn() { + channelz.RemoveEntry(t.channelzID) + } // Notify all active streams. for _, s := range streams { - s.mu.Lock() - if !s.headerDone { - close(s.headerChan) - s.headerDone = true - } - s.mu.Unlock() - s.write(recvMsg{err: ErrConnClosing}) + t.closeStream(s, ErrConnClosing, false, http2.ErrCodeNo, nil, nil, false) } - if stats.On() { + if t.statsHandler != nil { connEnd := &stats.ConnEnd{ Client: true, } - stats.HandleConn(t.ctx, connEnd) + t.statsHandler.HandleConn(t.ctx, connEnd) } - return + return err } +// GracefulClose sets the state to draining, which prevents new streams from +// being created and causes the transport to be closed when the last active +// stream is closed. If there are no active streams, the transport is closed +// immediately. This does nothing if the transport is already draining or +// closing. func (t *http2Client) GracefulClose() error { t.mu.Lock() - switch t.state { - case unreachable: - // The server may close the connection concurrently. t is not available for - // any streams. Close it now. - t.mu.Unlock() - t.Close() - return nil - case closing: - t.mu.Unlock() - return nil - } - // Notify the streams which were initiated after the server sent GOAWAY. - select { - case <-t.goAway: - n := t.prevGoAwayID - if n == 0 && t.nextID > 1 { - n = t.nextID - 2 - } - m := t.goAwayID + 2 - if m == 2 { - m = 1 - } - for i := m; i <= n; i += 2 { - if s, ok := t.activeStreams[i]; ok { - close(s.goAway) - } - } - default: - } - if t.state == draining { + // Make sure we move to draining only from active. + if t.state == draining || t.state == closing { t.mu.Unlock() return nil } @@ -614,104 +740,35 @@ func (t *http2Client) GracefulClose() error { // Write formats the data into HTTP2 data frame(s) and sends it out. The caller // should proceed only if Write returns nil. -// TODO(zhaoq): opts.Delay is ignored in this implementation. Support it later -// if it improves the performance. -func (t *http2Client) Write(s *Stream, data []byte, opts *Options) error { - r := bytes.NewBuffer(data) - for { - var p []byte - if r.Len() > 0 { - size := http2MaxFrameLen - // Wait until the stream has some quota to send the data. - sq, err := wait(s.ctx, s.done, s.goAway, t.shutdownChan, s.sendQuotaPool.acquire()) - if err != nil { - return err - } - // Wait until the transport has some quota to send the data. - tq, err := wait(s.ctx, s.done, s.goAway, t.shutdownChan, t.sendQuotaPool.acquire()) - if err != nil { - return err - } - if sq < size { - size = sq - } - if tq < size { - size = tq - } - p = r.Next(size) - ps := len(p) - if ps < sq { - // Overbooked stream quota. Return it back. - s.sendQuotaPool.add(sq - ps) - } - if ps < tq { - // Overbooked transport quota. Return it back. - t.sendQuotaPool.add(tq - ps) - } +func (t *http2Client) Write(s *Stream, hdr []byte, data []byte, opts *Options) error { + if opts.Last { + // If it's the last message, update stream state. + if !s.compareAndSwapState(streamActive, streamWriteDone) { + return errStreamDone } - var ( - endStream bool - forceFlush bool - ) - if opts.Last && r.Len() == 0 { - endStream = true + } else if s.getState() != streamActive { + return errStreamDone + } + df := &dataFrame{ + streamID: s.id, + endStream: opts.Last, + } + if hdr != nil || data != nil { // If it's not an empty data frame. + // Add some data to grpc message header so that we can equally + // distribute bytes across frames. + emptyLen := http2MaxFrameLen - len(hdr) + if emptyLen > len(data) { + emptyLen = len(data) } - // Indicate there is a writer who is about to write a data frame. - t.framer.adjustNumWriters(1) - // Got some quota. Try to acquire writing privilege on the transport. - if _, err := wait(s.ctx, s.done, s.goAway, t.shutdownChan, t.writableChan); err != nil { - if _, ok := err.(StreamError); ok || err == io.EOF { - // Return the connection quota back. - t.sendQuotaPool.add(len(p)) - } - if t.framer.adjustNumWriters(-1) == 0 { - // This writer is the last one in this batch and has the - // responsibility to flush the buffered frames. It queues - // a flush request to controlBuf instead of flushing directly - // in order to avoid the race with other writing or flushing. - t.controlBuf.put(&flushIO{}) - } + hdr = append(hdr, data[:emptyLen]...) + data = data[emptyLen:] + df.h, df.d = hdr, data + // TODO(mmukhi): The above logic in this if can be moved to loopyWriter's data handler. + if err := s.wq.get(int32(len(hdr) + len(data))); err != nil { return err } - select { - case <-s.ctx.Done(): - t.sendQuotaPool.add(len(p)) - if t.framer.adjustNumWriters(-1) == 0 { - t.controlBuf.put(&flushIO{}) - } - t.writableChan <- 0 - return ContextErr(s.ctx.Err()) - default: - } - if r.Len() == 0 && t.framer.adjustNumWriters(0) == 1 { - // Do a force flush iff this is last frame for the entire gRPC message - // and the caller is the only writer at this moment. - forceFlush = true - } - // If WriteData fails, all the pending streams will be handled - // by http2Client.Close(). No explicit CloseStream() needs to be - // invoked. - if err := t.framer.writeData(forceFlush, s.id, endStream, p); err != nil { - t.notifyError(err) - return connectionErrorf(true, err, "transport: %v", err) - } - if t.framer.adjustNumWriters(-1) == 0 { - t.framer.flushWrite() - } - t.writableChan <- 0 - if r.Len() == 0 { - break - } - } - if !opts.Last { - return nil } - s.mu.Lock() - if s.state != streamDone { - s.state = streamWriteDone - } - s.mu.Unlock() - return nil + return t.controlBuf.put(df) } func (t *http2Client) getStream(f http2.Frame) (*Stream, bool) { @@ -721,79 +778,110 @@ func (t *http2Client) getStream(f http2.Frame) (*Stream, bool) { return s, ok } -// updateWindow adjusts the inbound quota for the stream and the transport. -// Window updates will deliver to the controller for sending when -// the cumulative quota exceeds the corresponding threshold. +// adjustWindow sends out extra window update over the initial window size +// of stream if the application is requesting data larger in size than +// the window. +func (t *http2Client) adjustWindow(s *Stream, n uint32) { + if w := s.fc.maybeAdjust(n); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, increment: w}) + } +} + +// updateWindow adjusts the inbound quota for the stream. +// Window updates will be sent out when the cumulative quota +// exceeds the corresponding threshold. func (t *http2Client) updateWindow(s *Stream, n uint32) { - s.mu.Lock() - defer s.mu.Unlock() - if s.state == streamDone { - return + if w := s.fc.onRead(n); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, increment: w}) } - if w := t.fc.onRead(n); w > 0 { - t.controlBuf.put(&windowUpdate{0, w}) +} + +// updateFlowControl updates the incoming flow control windows +// for the transport and the stream based on the current bdp +// estimation. +func (t *http2Client) updateFlowControl(n uint32) { + t.mu.Lock() + for _, s := range t.activeStreams { + s.fc.newLimit(n) } - if w := s.fc.onRead(n); w > 0 { - t.controlBuf.put(&windowUpdate{s.id, w}) + t.mu.Unlock() + updateIWS := func(interface{}) bool { + t.initialWindowSize = int32(n) + return true } + t.controlBuf.executeAndPut(updateIWS, &outgoingWindowUpdate{streamID: 0, increment: t.fc.newLimit(n)}) + t.controlBuf.put(&outgoingSettings{ + ss: []http2.Setting{ + { + ID: http2.SettingInitialWindowSize, + Val: n, + }, + }, + }) } func (t *http2Client) handleData(f *http2.DataFrame) { - size := len(f.Data()) - if err := t.fc.onData(uint32(size)); err != nil { - t.notifyError(connectionErrorf(true, err, "%v", err)) - return + size := f.Header().Length + var sendBDPPing bool + if t.bdpEst != nil { + sendBDPPing = t.bdpEst.add(size) + } + // Decouple connection's flow control from application's read. + // An update on connection's flow control should not depend on + // whether user application has read the data or not. Such a + // restriction is already imposed on the stream's flow control, + // and therefore the sender will be blocked anyways. + // Decoupling the connection flow control will prevent other + // active(fast) streams from starving in presence of slow or + // inactive streams. + // + if w := t.fc.onData(size); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) + } + if sendBDPPing { + // Avoid excessive ping detection (e.g. in an L7 proxy) + // by sending a window update prior to the BDP ping. + + if w := t.fc.reset(); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) + } + + t.controlBuf.put(bdpPing) } // Select the right stream to dispatch. s, ok := t.getStream(f) if !ok { - if w := t.fc.onRead(uint32(size)); w > 0 { - t.controlBuf.put(&windowUpdate{0, w}) - } return } if size > 0 { - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - // The stream has been closed. Release the corresponding quota. - if w := t.fc.onRead(uint32(size)); w > 0 { - t.controlBuf.put(&windowUpdate{0, w}) - } + if err := s.fc.onData(size); err != nil { + t.closeStream(s, io.EOF, true, http2.ErrCodeFlowControl, status.New(codes.Internal, err.Error()), nil, false) return } - if err := s.fc.onData(uint32(size)); err != nil { - s.state = streamDone - s.statusCode = codes.Internal - s.statusDesc = err.Error() - close(s.done) - s.mu.Unlock() - s.write(recvMsg{err: io.EOF}) - t.controlBuf.put(&resetStream{s.id, http2.ErrCodeFlowControl}) - return + if f.Header().Flags.Has(http2.FlagDataPadded) { + if w := s.fc.onRead(size - uint32(len(f.Data()))); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{s.id, w}) + } } - s.mu.Unlock() // TODO(bradfitz, zhaoq): A copy is required here because there is no // guarantee f.Data() is consumed before the arrival of next frame. // Can this copy be eliminated? - data := make([]byte, size) - copy(data, f.Data()) - s.write(recvMsg{data: data}) + if len(f.Data()) > 0 { + data := make([]byte, len(f.Data())) + copy(data, f.Data()) + s.write(recvMsg{data: data}) + } } // The server has closed the stream without sending trailers. Record that // the read direction is closed, and set the status appropriately. if f.FrameHeader.Flags.Has(http2.FlagDataEndStream) { - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - return - } - s.state = streamDone - s.statusCode = codes.Internal - s.statusDesc = "server closed the stream without sending trailers" - close(s.done) - s.mu.Unlock() - s.write(recvMsg{err: io.EOF}) + t.closeStream(s, io.EOF, false, http2.ErrCodeNo, status.New(codes.Internal, "server closed the stream without sending trailers"), nil, true) } } @@ -802,42 +890,63 @@ func (t *http2Client) handleRSTStream(f *http2.RSTStreamFrame) { if !ok { return } - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - return - } - s.state = streamDone - if !s.headerDone { - close(s.headerChan) - s.headerDone = true + if f.ErrCode == http2.ErrCodeRefusedStream { + // The stream was unprocessed by the server. + atomic.StoreUint32(&s.unprocessed, 1) } - s.statusCode, ok = http2ErrConvTab[http2.ErrCode(f.ErrCode)] + statusCode, ok := http2ErrConvTab[f.ErrCode] if !ok { - grpclog.Println("transport: http2Client.handleRSTStream found no mapped gRPC status for the received http2 error ", f.ErrCode) - s.statusCode = codes.Unknown + warningf("transport: http2Client.handleRSTStream found no mapped gRPC status for the received http2 error %v", f.ErrCode) + statusCode = codes.Unknown } - s.statusDesc = fmt.Sprintf("stream terminated by RST_STREAM with error code: %d", f.ErrCode) - close(s.done) - s.mu.Unlock() - s.write(recvMsg{err: io.EOF}) + t.closeStream(s, io.EOF, false, http2.ErrCodeNo, status.Newf(statusCode, "stream terminated by RST_STREAM with error code: %v", f.ErrCode), nil, false) } -func (t *http2Client) handleSettings(f *http2.SettingsFrame) { +func (t *http2Client) handleSettings(f *http2.SettingsFrame, isFirst bool) { if f.IsAck() { return } + var maxStreams *uint32 var ss []http2.Setting f.ForeachSetting(func(s http2.Setting) error { + if s.ID == http2.SettingMaxConcurrentStreams { + maxStreams = new(uint32) + *maxStreams = s.Val + return nil + } ss = append(ss, s) return nil }) - // The settings will be applied once the ack is sent. - t.controlBuf.put(&settings{ack: true, ss: ss}) + if isFirst && maxStreams == nil { + maxStreams = new(uint32) + *maxStreams = math.MaxUint32 + } + sf := &incomingSettings{ + ss: ss, + } + if maxStreams == nil { + t.controlBuf.put(sf) + return + } + updateStreamQuota := func(interface{}) bool { + delta := int64(*maxStreams) - int64(t.maxConcurrentStreams) + t.maxConcurrentStreams = *maxStreams + t.streamQuota += delta + if delta > 0 && t.waitingStreams > 0 { + close(t.streamsQuotaAvailable) // wake all of them up. + t.streamsQuotaAvailable = make(chan struct{}, 1) + } + return true + } + t.controlBuf.executeAndPut(updateStreamQuota, sf) } func (t *http2Client) handlePing(f *http2.PingFrame) { - if f.IsAck() { // Do nothing. + if f.IsAck() { + // Maybe it's a BDP ping. + if t.bdpEst != nil { + t.bdpEst.calculate(f.Data) + } return } pingAck := &ping{ack: true} @@ -847,63 +956,101 @@ func (t *http2Client) handlePing(f *http2.PingFrame) { func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { t.mu.Lock() - if t.state == reachable || t.state == draining { - if f.LastStreamID > 0 && f.LastStreamID%2 != 1 { - t.mu.Unlock() - t.notifyError(connectionErrorf(true, nil, "received illegal http2 GOAWAY frame: stream ID %d is even", f.LastStreamID)) - return - } - select { - case <-t.goAway: - id := t.goAwayID - // t.goAway has been closed (i.e.,multiple GoAways). - if id < f.LastStreamID { - t.mu.Unlock() - t.notifyError(connectionErrorf(true, nil, "received illegal http2 GOAWAY frame: previously recv GOAWAY frame with LastStramID %d, currently recv %d", id, f.LastStreamID)) - return - } - t.prevGoAwayID = id - t.goAwayID = f.LastStreamID + if t.state == closing { + t.mu.Unlock() + return + } + if f.ErrCode == http2.ErrCodeEnhanceYourCalm { + infof("Client received GoAway with http2.ErrCodeEnhanceYourCalm.") + } + id := f.LastStreamID + if id > 0 && id%2 != 1 { + t.mu.Unlock() + t.Close() + return + } + // A client can receive multiple GoAways from the server (see + // https://github.com/grpc/grpc-go/issues/1387). The idea is that the first + // GoAway will be sent with an ID of MaxInt32 and the second GoAway will be + // sent after an RTT delay with the ID of the last stream the server will + // process. + // + // Therefore, when we get the first GoAway we don't necessarily close any + // streams. While in case of second GoAway we close all streams created after + // the GoAwayId. This way streams that were in-flight while the GoAway from + // server was being sent don't get killed. + select { + case <-t.goAway: // t.goAway has been closed (i.e.,multiple GoAways). + // If there are multiple GoAways the first one should always have an ID greater than the following ones. + if id > t.prevGoAwayID { t.mu.Unlock() + t.Close() return - default: } - t.goAwayID = f.LastStreamID + default: + t.setGoAwayReason(f) close(t.goAway) + t.state = draining + t.controlBuf.put(&incomingGoAway{}) + } + // All streams with IDs greater than the GoAwayId + // and smaller than the previous GoAway ID should be killed. + upperLimit := t.prevGoAwayID + if upperLimit == 0 { // This is the first GoAway Frame. + upperLimit = math.MaxUint32 // Kill all streams after the GoAway ID. + } + for streamID, stream := range t.activeStreams { + if streamID > id && streamID <= upperLimit { + // The stream was unprocessed by the server. + atomic.StoreUint32(&stream.unprocessed, 1) + t.closeStream(stream, errStreamDrain, false, http2.ErrCodeNo, statusGoAway, nil, false) + } } + t.prevGoAwayID = id + active := len(t.activeStreams) t.mu.Unlock() + if active == 0 { + t.Close() + } } -func (t *http2Client) handleWindowUpdate(f *http2.WindowUpdateFrame) { - id := f.Header().StreamID - incr := f.Increment - if id == 0 { - t.sendQuotaPool.add(int(incr)) - return - } - if s, ok := t.getStream(f); ok { - s.sendQuotaPool.add(int(incr)) +// setGoAwayReason sets the value of t.goAwayReason based +// on the GoAway frame received. +// It expects a lock on transport's mutext to be held by +// the caller. +func (t *http2Client) setGoAwayReason(f *http2.GoAwayFrame) { + t.goAwayReason = GoAwayNoReason + switch f.ErrCode { + case http2.ErrCodeEnhanceYourCalm: + if string(f.DebugData()) == "too_many_pings" { + t.goAwayReason = GoAwayTooManyPings + } } } +func (t *http2Client) GetGoAwayReason() GoAwayReason { + t.mu.Lock() + defer t.mu.Unlock() + return t.goAwayReason +} + +func (t *http2Client) handleWindowUpdate(f *http2.WindowUpdateFrame) { + t.controlBuf.put(&incomingWindowUpdate{ + streamID: f.Header().StreamID, + increment: f.Increment, + }) +} + // operateHeaders takes action on the decoded headers. func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) { s, ok := t.getStream(frame) if !ok { return } + atomic.StoreUint32(&s.bytesReceived, 1) var state decodeState - for _, hf := range frame.Fields { - state.processHeaderField(hf) - } - if state.err != nil { - s.mu.Lock() - if !s.headerDone { - close(s.headerChan) - s.headerDone = true - } - s.mu.Unlock() - s.write(recvMsg{err: state.err}) + if err := state.decodeResponseHeader(frame); err != nil { + t.closeStream(s, err, true, http2.ErrCodeProtocol, nil, nil, false) // Something wrong. Stops reading even when there is remaining. return } @@ -911,59 +1058,41 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) { endStream := frame.StreamEnded() var isHeader bool defer func() { - if stats.On() { + if t.statsHandler != nil { if isHeader { inHeader := &stats.InHeader{ Client: true, WireLength: int(frame.Header().Length), } - stats.HandleRPC(s.clientStatsCtx, inHeader) + t.statsHandler.HandleRPC(s.ctx, inHeader) } else { inTrailer := &stats.InTrailer{ Client: true, WireLength: int(frame.Header().Length), } - stats.HandleRPC(s.clientStatsCtx, inTrailer) + t.statsHandler.HandleRPC(s.ctx, inTrailer) } } }() - - s.mu.Lock() - if !endStream { - s.recvCompress = state.encoding - } - if !s.headerDone { - if !endStream && len(state.mdata) > 0 { - s.header = state.mdata + // If headers haven't been received yet. + if atomic.SwapUint32(&s.headerDone, 1) == 0 { + if !endStream { + // Headers frame is not actually a trailers-only frame. + isHeader = true + // These values can be set without any synchronization because + // stream goroutine will read it only after seeing a closed + // headerChan which we'll close after setting this. + s.recvCompress = state.encoding + if len(state.mdata) > 0 { + s.header = state.mdata + } } close(s.headerChan) - s.headerDone = true - isHeader = true } - if !endStream || s.state == streamDone { - s.mu.Unlock() + if !endStream { return } - - if len(state.mdata) > 0 { - s.trailer = state.mdata - } - s.statusCode = state.statusCode - s.statusDesc = state.statusDesc - close(s.done) - s.state = streamDone - s.mu.Unlock() - s.write(recvMsg{err: io.EOF}) -} - -func handleMalformedHTTP2(s *Stream, err error) { - s.mu.Lock() - if !s.headerDone { - close(s.headerChan) - s.headerDone = true - } - s.mu.Unlock() - s.write(recvMsg{err: err}) + t.closeStream(s, io.EOF, false, http2.ErrCodeNo, state.status(), state.mdata, true) } // reader runs as a separate goroutine in charge of reading data from network @@ -973,22 +1102,26 @@ func handleMalformedHTTP2(s *Stream, err error) { // optimal. // TODO(zhaoq): Check the validity of the incoming frame sequence. func (t *http2Client) reader() { + defer close(t.readerDone) // Check the validity of server preface. - frame, err := t.framer.readFrame() + frame, err := t.framer.fr.ReadFrame() if err != nil { - t.notifyError(err) + t.Close() return } + atomic.CompareAndSwapUint32(&t.activity, 0, 1) sf, ok := frame.(*http2.SettingsFrame) if !ok { - t.notifyError(err) + t.Close() return } - t.handleSettings(sf) + t.onSuccess() + t.handleSettings(sf, true) // loop to keep reading incoming messages on this transport. for { - frame, err := t.framer.readFrame() + frame, err := t.framer.fr.ReadFrame() + atomic.CompareAndSwapUint32(&t.activity, 0, 1) if err != nil { // Abort an active stream if the http2.Framer returns a // http2.StreamError. This can happen only if the server's response @@ -999,12 +1132,12 @@ func (t *http2Client) reader() { t.mu.Unlock() if s != nil { // use error detail to provide better err message - handleMalformedHTTP2(s, streamErrorf(http2ErrConvTab[se.Code], "%v", t.framer.errorDetail())) + t.closeStream(s, streamErrorf(http2ErrConvTab[se.Code], "%v", t.framer.fr.ErrorDetail()), true, http2.ErrCodeProtocol, nil, nil, false) } continue } else { // Transport error. - t.notifyError(err) + t.Close() return } } @@ -1016,7 +1149,7 @@ func (t *http2Client) reader() { case *http2.RSTStreamFrame: t.handleRSTStream(frame) case *http2.SettingsFrame: - t.handleSettings(frame) + t.handleSettings(frame, false) case *http2.PingFrame: t.handlePing(frame) case *http2.GoAwayFrame: @@ -1024,103 +1157,128 @@ func (t *http2Client) reader() { case *http2.WindowUpdateFrame: t.handleWindowUpdate(frame) default: - grpclog.Printf("transport: http2Client.reader got unhandled frame type %v.", frame) + errorf("transport: http2Client.reader got unhandled frame type %v.", frame) } } } -func (t *http2Client) applySettings(ss []http2.Setting) { - for _, s := range ss { - switch s.ID { - case http2.SettingMaxConcurrentStreams: - // TODO(zhaoq): This is a hack to avoid significant refactoring of the - // code to deal with the unrealistic int32 overflow. Probably will try - // to find a better way to handle this later. - if s.Val > math.MaxInt32 { - s.Val = math.MaxInt32 - } - t.mu.Lock() - reset := t.streamsQuota != nil - if !reset { - t.streamsQuota = newQuotaPool(int(s.Val) - len(t.activeStreams)) - } - ms := t.maxStreams - t.maxStreams = int(s.Val) - t.mu.Unlock() - if reset { - t.streamsQuota.add(int(s.Val) - ms) +// keepalive running in a separate goroutune makes sure the connection is alive by sending pings. +func (t *http2Client) keepalive() { + p := &ping{data: [8]byte{}} + timer := time.NewTimer(t.kp.Time) + for { + select { + case <-timer.C: + if atomic.CompareAndSwapUint32(&t.activity, 1, 0) { + timer.Reset(t.kp.Time) + continue } - case http2.SettingInitialWindowSize: + // Check if keepalive should go dormant. t.mu.Lock() - for _, stream := range t.activeStreams { - // Adjust the sending quota for each stream. - stream.sendQuotaPool.add(int(s.Val - t.streamSendQuota)) + if len(t.activeStreams) < 1 && !t.kp.PermitWithoutStream { + // Make awakenKeepalive writable. + <-t.awakenKeepalive + t.mu.Unlock() + select { + case <-t.awakenKeepalive: + // If the control gets here a ping has been sent + // need to reset the timer with keepalive.Timeout. + case <-t.ctx.Done(): + return + } + } else { + t.mu.Unlock() + if channelz.IsOn() { + t.czmu.Lock() + t.kpCount++ + t.czmu.Unlock() + } + // Send ping. + t.controlBuf.put(p) } - t.streamSendQuota = s.Val - t.mu.Unlock() - } - } -} -// controller running in a separate goroutine takes charge of sending control -// frames (e.g., window update, reset stream, setting, etc.) to the server. -func (t *http2Client) controller() { - for { - select { - case i := <-t.controlBuf.get(): - t.controlBuf.load() + // By the time control gets here a ping has been sent one way or the other. + timer.Reset(t.kp.Timeout) select { - case <-t.writableChan: - switch i := i.(type) { - case *windowUpdate: - t.framer.writeWindowUpdate(true, i.streamID, i.increment) - case *settings: - if i.ack { - t.framer.writeSettingsAck(true) - t.applySettings(i.ss) - } else { - t.framer.writeSettings(true, i.ss...) - } - case *resetStream: - t.framer.writeRSTStream(true, i.streamID, i.code) - case *flushIO: - t.framer.flushWrite() - case *ping: - t.framer.writePing(true, i.ack, i.data) - default: - grpclog.Printf("transport: http2Client.controller got unexpected item type %v\n", i) + case <-timer.C: + if atomic.CompareAndSwapUint32(&t.activity, 1, 0) { + timer.Reset(t.kp.Time) + continue } - t.writableChan <- 0 - continue - case <-t.shutdownChan: + t.Close() return + case <-t.ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return + } + case <-t.ctx.Done(): + if !timer.Stop() { + <-timer.C } - case <-t.shutdownChan: return } } } func (t *http2Client) Error() <-chan struct{} { - return t.errorChan + return t.ctx.Done() } func (t *http2Client) GoAway() <-chan struct{} { return t.goAway } -func (t *http2Client) notifyError(err error) { - t.mu.Lock() - // make sure t.errorChan is closed only once. - if t.state == draining { - t.mu.Unlock() - t.Close() - return - } - if t.state == reachable { - t.state = unreachable - close(t.errorChan) - grpclog.Printf("transport: http2Client.notifyError got notified that the client transport was broken %v.", err) +func (t *http2Client) ChannelzMetric() *channelz.SocketInternalMetric { + t.czmu.RLock() + s := channelz.SocketInternalMetric{ + StreamsStarted: t.streamsStarted, + StreamsSucceeded: t.streamsSucceeded, + StreamsFailed: t.streamsFailed, + MessagesSent: t.msgSent, + MessagesReceived: t.msgRecv, + KeepAlivesSent: t.kpCount, + LastLocalStreamCreatedTimestamp: t.lastStreamCreated, + LastMessageSentTimestamp: t.lastMsgSent, + LastMessageReceivedTimestamp: t.lastMsgRecv, + LocalFlowControlWindow: int64(t.fc.getSize()), + //socket options + LocalAddr: t.localAddr, + RemoteAddr: t.remoteAddr, + // Security + // RemoteName : + } + t.czmu.RUnlock() + s.RemoteFlowControlWindow = t.getOutFlowWindow() + return &s +} + +func (t *http2Client) IncrMsgSent() { + t.czmu.Lock() + t.msgSent++ + t.lastMsgSent = time.Now() + t.czmu.Unlock() +} + +func (t *http2Client) IncrMsgRecv() { + t.czmu.Lock() + t.msgRecv++ + t.lastMsgRecv = time.Now() + t.czmu.Unlock() +} + +func (t *http2Client) getOutFlowWindow() int64 { + resp := make(chan uint32, 1) + timer := time.NewTimer(time.Second) + defer timer.Stop() + t.controlBuf.put(&outFlowControlSizeRequest{resp}) + select { + case sz := <-resp: + return int64(sz) + case <-t.ctxDone: + return -1 + case <-timer.C: + return -2 } - t.mu.Unlock() } diff --git a/vendor/google.golang.org/grpc/transport/http2_server.go b/vendor/google.golang.org/grpc/transport/http2_server.go index 316188e7b78..8b93e222e9e 100644 --- a/vendor/google.golang.org/grpc/transport/http2_server.go +++ b/vendor/google.golang.org/grpc/transport/http2_server.go @@ -1,33 +1,18 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -36,21 +21,29 @@ package transport import ( "bytes" "errors" + "fmt" "io" "math" + "math/rand" "net" "strconv" "sync" + "sync/atomic" + "time" + "github.com/golang/protobuf/proto" "golang.org/x/net/context" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" + + "google.golang.org/grpc/channelz" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" - "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" "google.golang.org/grpc/tap" ) @@ -61,131 +54,265 @@ var ErrIllegalHeaderWrite = errors.New("transport: the stream is done or WriteHe // http2Server implements the ServerTransport interface with HTTP2. type http2Server struct { ctx context.Context + ctxDone <-chan struct{} // Cache the context.Done() chan + cancel context.CancelFunc conn net.Conn + loopy *loopyWriter + readerDone chan struct{} // sync point to enable testing. + writerDone chan struct{} // sync point to enable testing. remoteAddr net.Addr localAddr net.Addr maxStreamID uint32 // max stream ID ever seen authInfo credentials.AuthInfo // auth info about the connection inTapHandle tap.ServerInHandle - // writableChan synchronizes write access to the transport. - // A writer acquires the write lock by receiving a value on writableChan - // and releases it by sending on writableChan. - writableChan chan int - // shutdownChan is closed when Close is called. - // Blocking operations should select on shutdownChan to avoid - // blocking forever after Close. - shutdownChan chan struct{} - framer *framer - hBuf *bytes.Buffer // the buffer for HPACK encoding - hEnc *hpack.Encoder // HPACK encoder - + framer *framer // The max number of concurrent streams. maxStreams uint32 // controlBuf delivers all the control related tasks (e.g., window // updates, reset streams, and various settings) to the controller. - controlBuf *recvBuffer - fc *inFlow - // sendQuotaPool provides flow control to outbound message. - sendQuotaPool *quotaPool + controlBuf *controlBuffer + fc *trInFlow + stats stats.Handler + // Flag to keep track of reading activity on transport. + // 1 is true and 0 is false. + activity uint32 // Accessed atomically. + // Keepalive and max-age parameters for the server. + kp keepalive.ServerParameters + + // Keepalive enforcement policy. + kep keepalive.EnforcementPolicy + // The time instance last ping was received. + lastPingAt time.Time + // Number of times the client has violated keepalive ping policy so far. + pingStrikes uint8 + // Flag to signify that number of ping strikes should be reset to 0. + // This is set whenever data or header frames are sent. + // 1 means yes. + resetPingStrikes uint32 // Accessed atomically. + initialWindowSize int32 + bdpEst *bdpEstimator + + mu sync.Mutex // guard the following - mu sync.Mutex // guard the following + // drainChan is initialized when drain(...) is called the first time. + // After which the server writes out the first GoAway(with ID 2^31-1) frame. + // Then an independent goroutine will be launched to later send the second GoAway. + // During this time we don't want to write another first GoAway(with ID 2^31 -1) frame. + // Thus call to drain(...) will be a no-op if drainChan is already initialized since draining is + // already underway. + drainChan chan struct{} state transportState activeStreams map[uint32]*Stream - // the per-stream outbound flow control window size set by the peer. - streamSendQuota uint32 + // idle is the time instant when the connection went idle. + // This is either the beginning of the connection or when the number of + // RPCs go down to 0. + // When the connection is busy, this value is set to 0. + idle time.Time + + // Fields below are for channelz metric collection. + channelzID int64 // channelz unique identification number + czmu sync.RWMutex + kpCount int64 + // The number of streams that have started, including already finished ones. + streamsStarted int64 + // The number of streams that have ended successfully by sending frame with + // EoS bit set. + streamsSucceeded int64 + streamsFailed int64 + lastStreamCreated time.Time + msgSent int64 + msgRecv int64 + lastMsgSent time.Time + lastMsgRecv time.Time } // newHTTP2Server constructs a ServerTransport based on HTTP2. ConnectionError is // returned if something goes wrong. func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err error) { - framer := newFramer(conn) + writeBufSize := defaultWriteBufSize + if config.WriteBufferSize > 0 { + writeBufSize = config.WriteBufferSize + } + readBufSize := defaultReadBufSize + if config.ReadBufferSize > 0 { + readBufSize = config.ReadBufferSize + } + framer := newFramer(conn, writeBufSize, readBufSize) // Send initial settings as connection preface to client. - var settings []http2.Setting + var isettings []http2.Setting // TODO(zhaoq): Have a better way to signal "no limit" because 0 is // permitted in the HTTP2 spec. maxStreams := config.MaxStreams if maxStreams == 0 { maxStreams = math.MaxUint32 } else { - settings = append(settings, http2.Setting{ + isettings = append(isettings, http2.Setting{ ID: http2.SettingMaxConcurrentStreams, Val: maxStreams, }) } - if initialWindowSize != defaultWindowSize { - settings = append(settings, http2.Setting{ + dynamicWindow := true + iwz := int32(initialWindowSize) + if config.InitialWindowSize >= defaultWindowSize { + iwz = config.InitialWindowSize + dynamicWindow = false + } + icwz := int32(initialWindowSize) + if config.InitialConnWindowSize >= defaultWindowSize { + icwz = config.InitialConnWindowSize + dynamicWindow = false + } + if iwz != defaultWindowSize { + isettings = append(isettings, http2.Setting{ ID: http2.SettingInitialWindowSize, - Val: uint32(initialWindowSize)}) + Val: uint32(iwz)}) } - if err := framer.writeSettings(true, settings...); err != nil { - return nil, connectionErrorf(true, err, "transport: %v", err) + if err := framer.fr.WriteSettings(isettings...); err != nil { + return nil, connectionErrorf(false, err, "transport: %v", err) } // Adjust the connection flow control window if needed. - if delta := uint32(initialConnWindowSize - defaultWindowSize); delta > 0 { - if err := framer.writeWindowUpdate(true, 0, delta); err != nil { - return nil, connectionErrorf(true, err, "transport: %v", err) + if delta := uint32(icwz - defaultWindowSize); delta > 0 { + if err := framer.fr.WriteWindowUpdate(0, delta); err != nil { + return nil, connectionErrorf(false, err, "transport: %v", err) } } - var buf bytes.Buffer + kp := config.KeepaliveParams + if kp.MaxConnectionIdle == 0 { + kp.MaxConnectionIdle = defaultMaxConnectionIdle + } + if kp.MaxConnectionAge == 0 { + kp.MaxConnectionAge = defaultMaxConnectionAge + } + // Add a jitter to MaxConnectionAge. + kp.MaxConnectionAge += getJitter(kp.MaxConnectionAge) + if kp.MaxConnectionAgeGrace == 0 { + kp.MaxConnectionAgeGrace = defaultMaxConnectionAgeGrace + } + if kp.Time == 0 { + kp.Time = defaultServerKeepaliveTime + } + if kp.Timeout == 0 { + kp.Timeout = defaultServerKeepaliveTimeout + } + kep := config.KeepalivePolicy + if kep.MinTime == 0 { + kep.MinTime = defaultKeepalivePolicyMinTime + } + ctx, cancel := context.WithCancel(context.Background()) t := &http2Server{ - ctx: context.Background(), - conn: conn, - remoteAddr: conn.RemoteAddr(), - localAddr: conn.LocalAddr(), - authInfo: config.AuthInfo, - framer: framer, - hBuf: &buf, - hEnc: hpack.NewEncoder(&buf), - maxStreams: maxStreams, - inTapHandle: config.InTapHandle, - controlBuf: newRecvBuffer(), - fc: &inFlow{limit: initialConnWindowSize}, - sendQuotaPool: newQuotaPool(defaultWindowSize), - state: reachable, - writableChan: make(chan int, 1), - shutdownChan: make(chan struct{}), - activeStreams: make(map[uint32]*Stream), - streamSendQuota: defaultWindowSize, - } - if stats.On() { - t.ctx = stats.TagConn(t.ctx, &stats.ConnTagInfo{ + ctx: ctx, + cancel: cancel, + ctxDone: ctx.Done(), + conn: conn, + remoteAddr: conn.RemoteAddr(), + localAddr: conn.LocalAddr(), + authInfo: config.AuthInfo, + framer: framer, + readerDone: make(chan struct{}), + writerDone: make(chan struct{}), + maxStreams: maxStreams, + inTapHandle: config.InTapHandle, + fc: &trInFlow{limit: uint32(icwz)}, + state: reachable, + activeStreams: make(map[uint32]*Stream), + stats: config.StatsHandler, + kp: kp, + idle: time.Now(), + kep: kep, + initialWindowSize: iwz, + } + t.controlBuf = newControlBuffer(t.ctxDone) + if dynamicWindow { + t.bdpEst = &bdpEstimator{ + bdp: initialWindowSize, + updateFlowControl: t.updateFlowControl, + } + } + if t.stats != nil { + t.ctx = t.stats.TagConn(t.ctx, &stats.ConnTagInfo{ RemoteAddr: t.remoteAddr, LocalAddr: t.localAddr, }) connBegin := &stats.ConnBegin{} - stats.HandleConn(t.ctx, connBegin) + t.stats.HandleConn(t.ctx, connBegin) + } + if channelz.IsOn() { + t.channelzID = channelz.RegisterNormalSocket(t, config.ChannelzParentID, "") + } + t.framer.writer.Flush() + + defer func() { + if err != nil { + t.Close() + } + }() + + // Check the validity of client preface. + preface := make([]byte, len(clientPreface)) + if _, err := io.ReadFull(t.conn, preface); err != nil { + return nil, connectionErrorf(false, err, "transport: http2Server.HandleStreams failed to receive the preface from client: %v", err) + } + if !bytes.Equal(preface, clientPreface) { + return nil, connectionErrorf(false, nil, "transport: http2Server.HandleStreams received bogus greeting from client: %q", preface) + } + + frame, err := t.framer.fr.ReadFrame() + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil, err + } + if err != nil { + return nil, connectionErrorf(false, err, "transport: http2Server.HandleStreams failed to read initial settings frame: %v", err) } - go t.controller() - t.writableChan <- 0 + atomic.StoreUint32(&t.activity, 1) + sf, ok := frame.(*http2.SettingsFrame) + if !ok { + return nil, connectionErrorf(false, nil, "transport: http2Server.HandleStreams saw invalid preface type %T from client", frame) + } + t.handleSettings(sf) + + go func() { + t.loopy = newLoopyWriter(serverSide, t.framer, t.controlBuf, t.bdpEst) + t.loopy.ssGoAwayHandler = t.outgoingGoAwayHandler + t.loopy.run() + t.conn.Close() + close(t.writerDone) + }() + go t.keepalive() return t, nil } // operateHeader takes action on the decoded headers. func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(*Stream), traceCtx func(context.Context, string) context.Context) (close bool) { - buf := newRecvBuffer() - s := &Stream{ - id: frame.Header().StreamID, - st: t, - buf: buf, - fc: &inFlow{limit: initialWindowSize}, - } - + streamID := frame.Header().StreamID var state decodeState for _, hf := range frame.Fields { - state.processHeaderField(hf) - } - if err := state.err; err != nil { - if se, ok := err.(StreamError); ok { - t.controlBuf.put(&resetStream{s.id, statusCodeConvTab[se.Code]}) + if err := state.processHeaderField(hf); err != nil { + if se, ok := err.(StreamError); ok { + t.controlBuf.put(&cleanupStream{ + streamID: streamID, + rst: true, + rstCode: statusCodeConvTab[se.Code], + onWrite: func() {}, + }) + } + return } - return } + buf := newRecvBuffer() + s := &Stream{ + id: streamID, + st: t, + buf: buf, + fc: &inFlow{limit: uint32(t.initialWindowSize)}, + recvCompress: state.encoding, + method: state.method, + contentSubtype: state.contentSubtype, + } if frame.StreamEnded() { // s is just created by the caller. No lock needed. s.state = streamReadDone } - s.recvCompress = state.encoding if state.timeoutSet { s.ctx, s.cancel = context.WithTimeout(t.ctx, state.timeout) } else { @@ -199,21 +326,16 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( pr.AuthInfo = t.authInfo } s.ctx = peer.NewContext(s.ctx, pr) - // Cache the current stream to the context so that the server application - // can find out. Required when the server wants to send some metadata - // back to the client (unary call only). - s.ctx = newContextWithStream(s.ctx, s) // Attach the received metadata to the context. if len(state.mdata) > 0 { - s.ctx = metadata.NewContext(s.ctx, state.mdata) + s.ctx = metadata.NewIncomingContext(s.ctx, state.mdata) } - - s.dec = &recvBufferReader{ - ctx: s.ctx, - recv: s.buf, + if state.statsTags != nil { + s.ctx = stats.SetIncomingTags(s.ctx, state.statsTags) + } + if state.statsTrace != nil { + s.ctx = stats.SetIncomingTrace(s.ctx, state.statsTrace) } - s.recvCompress = state.encoding - s.method = state.method if t.inTapHandle != nil { var err error info := &tap.Info{ @@ -221,8 +343,13 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( } s.ctx, err = t.inTapHandle(s.ctx, info) if err != nil { - // TODO: Log the real error. - t.controlBuf.put(&resetStream{s.id, http2.ErrCodeRefusedStream}) + warningf("transport: http2Server.operateHeaders got an error from InTapHandle: %v", err) + t.controlBuf.put(&cleanupStream{ + streamID: s.id, + rst: true, + rstCode: http2.ErrCodeRefusedStream, + onWrite: func() {}, + }) return } } @@ -233,25 +360,38 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( } if uint32(len(t.activeStreams)) >= t.maxStreams { t.mu.Unlock() - t.controlBuf.put(&resetStream{s.id, http2.ErrCodeRefusedStream}) + t.controlBuf.put(&cleanupStream{ + streamID: streamID, + rst: true, + rstCode: http2.ErrCodeRefusedStream, + onWrite: func() {}, + }) return } - if s.id%2 != 1 || s.id <= t.maxStreamID { + if streamID%2 != 1 || streamID <= t.maxStreamID { t.mu.Unlock() // illegal gRPC stream id. - grpclog.Println("transport: http2Server.HandleStreams received an illegal stream id: ", s.id) + errorf("transport: http2Server.HandleStreams received an illegal stream id: %v", streamID) return true } - t.maxStreamID = s.id - s.sendQuotaPool = newQuotaPool(int(t.streamSendQuota)) - t.activeStreams[s.id] = s + t.maxStreamID = streamID + t.activeStreams[streamID] = s + if len(t.activeStreams) == 1 { + t.idle = time.Time{} + } t.mu.Unlock() - s.windowHandler = func(n int) { - t.updateWindow(s, uint32(n)) + if channelz.IsOn() { + t.czmu.Lock() + t.streamsStarted++ + t.lastStreamCreated = time.Now() + t.czmu.Unlock() + } + s.requestRead = func(n int) { + t.adjustWindow(s, uint32(n)) } s.ctx = traceCtx(s.ctx, s.method) - if stats.On() { - s.ctx = stats.TagRPC(s.ctx, &stats.RPCTagInfo{FullMethodName: s.method}) + if t.stats != nil { + s.ctx = t.stats.TagRPC(s.ctx, &stats.RPCTagInfo{FullMethodName: s.method}) inHeader := &stats.InHeader{ FullMethod: s.method, RemoteAddr: t.remoteAddr, @@ -259,7 +399,19 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( Compression: s.recvCompress, WireLength: int(frame.Header().Length), } - stats.HandleRPC(s.ctx, inHeader) + t.stats.HandleRPC(s.ctx, inHeader) + } + s.ctxDone = s.ctx.Done() + s.wq = newWriteQuota(defaultWriteQuota, s.ctxDone) + s.trReader = &transportReader{ + reader: &recvBufferReader{ + ctx: s.ctx, + ctxDone: s.ctxDone, + recv: s.buf, + }, + windowHandler: func(n int) { + t.updateWindow(s, uint32(n)) + }, } handle(s) return @@ -269,55 +421,33 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( // typically run in a separate goroutine. // traceCtx attaches trace to ctx and returns the new context. func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context.Context, string) context.Context) { - // Check the validity of client preface. - preface := make([]byte, len(clientPreface)) - if _, err := io.ReadFull(t.conn, preface); err != nil { - grpclog.Printf("transport: http2Server.HandleStreams failed to receive the preface from client: %v", err) - t.Close() - return - } - if !bytes.Equal(preface, clientPreface) { - grpclog.Printf("transport: http2Server.HandleStreams received bogus greeting from client: %q", preface) - t.Close() - return - } - - frame, err := t.framer.readFrame() - if err == io.EOF || err == io.ErrUnexpectedEOF { - t.Close() - return - } - if err != nil { - grpclog.Printf("transport: http2Server.HandleStreams failed to read frame: %v", err) - t.Close() - return - } - sf, ok := frame.(*http2.SettingsFrame) - if !ok { - grpclog.Printf("transport: http2Server.HandleStreams saw invalid preface type %T from client", frame) - t.Close() - return - } - t.handleSettings(sf) - + defer close(t.readerDone) for { - frame, err := t.framer.readFrame() + frame, err := t.framer.fr.ReadFrame() + atomic.StoreUint32(&t.activity, 1) if err != nil { if se, ok := err.(http2.StreamError); ok { + warningf("transport: http2Server.HandleStreams encountered http2.StreamError: %v", se) t.mu.Lock() s := t.activeStreams[se.StreamID] t.mu.Unlock() if s != nil { - t.closeStream(s) + t.closeStream(s, true, se.Code, nil, false) + } else { + t.controlBuf.put(&cleanupStream{ + streamID: se.StreamID, + rst: true, + rstCode: se.Code, + onWrite: func() {}, + }) } - t.controlBuf.put(&resetStream{se.StreamID, se.Code}) continue } if err == io.EOF || err == io.ErrUnexpectedEOF { t.Close() return } - grpclog.Printf("transport: http2Server.HandleStreams failed to read frame: %v", err) + warningf("transport: http2Server.HandleStreams failed to read frame: %v", err) t.Close() return } @@ -340,7 +470,7 @@ func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context. case *http2.GoAwayFrame: // TODO: Handle GoAway from the client appropriately. default: - grpclog.Printf("transport: http2Server.HandleStreams found unhandled frame type %v.", frame) + errorf("transport: http2Server.HandleStreams found unhandled frame type %v.", frame) } } } @@ -360,69 +490,110 @@ func (t *http2Server) getStream(f http2.Frame) (*Stream, bool) { return s, true } +// adjustWindow sends out extra window update over the initial window size +// of stream if the application is requesting data larger in size than +// the window. +func (t *http2Server) adjustWindow(s *Stream, n uint32) { + if w := s.fc.maybeAdjust(n); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, increment: w}) + } + +} + // updateWindow adjusts the inbound quota for the stream and the transport. // Window updates will deliver to the controller for sending when // the cumulative quota exceeds the corresponding threshold. func (t *http2Server) updateWindow(s *Stream, n uint32) { - s.mu.Lock() - defer s.mu.Unlock() - if s.state == streamDone { - return - } - if w := t.fc.onRead(n); w > 0 { - t.controlBuf.put(&windowUpdate{0, w}) - } if w := s.fc.onRead(n); w > 0 { - t.controlBuf.put(&windowUpdate{s.id, w}) + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, + increment: w, + }) } } +// updateFlowControl updates the incoming flow control windows +// for the transport and the stream based on the current bdp +// estimation. +func (t *http2Server) updateFlowControl(n uint32) { + t.mu.Lock() + for _, s := range t.activeStreams { + s.fc.newLimit(n) + } + t.initialWindowSize = int32(n) + t.mu.Unlock() + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: t.fc.newLimit(n), + }) + t.controlBuf.put(&outgoingSettings{ + ss: []http2.Setting{ + { + ID: http2.SettingInitialWindowSize, + Val: n, + }, + }, + }) + +} + func (t *http2Server) handleData(f *http2.DataFrame) { - size := len(f.Data()) - if err := t.fc.onData(uint32(size)); err != nil { - grpclog.Printf("transport: http2Server %v", err) - t.Close() - return + size := f.Header().Length + var sendBDPPing bool + if t.bdpEst != nil { + sendBDPPing = t.bdpEst.add(size) + } + // Decouple connection's flow control from application's read. + // An update on connection's flow control should not depend on + // whether user application has read the data or not. Such a + // restriction is already imposed on the stream's flow control, + // and therefore the sender will be blocked anyways. + // Decoupling the connection flow control will prevent other + // active(fast) streams from starving in presence of slow or + // inactive streams. + if w := t.fc.onData(size); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) + } + if sendBDPPing { + // Avoid excessive ping detection (e.g. in an L7 proxy) + // by sending a window update prior to the BDP ping. + if w := t.fc.reset(); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) + } + t.controlBuf.put(bdpPing) } // Select the right stream to dispatch. s, ok := t.getStream(f) if !ok { - if w := t.fc.onRead(uint32(size)); w > 0 { - t.controlBuf.put(&windowUpdate{0, w}) - } return } if size > 0 { - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - // The stream has been closed. Release the corresponding quota. - if w := t.fc.onRead(uint32(size)); w > 0 { - t.controlBuf.put(&windowUpdate{0, w}) - } + if err := s.fc.onData(size); err != nil { + t.closeStream(s, true, http2.ErrCodeFlowControl, nil, false) return } - if err := s.fc.onData(uint32(size)); err != nil { - s.mu.Unlock() - t.closeStream(s) - t.controlBuf.put(&resetStream{s.id, http2.ErrCodeFlowControl}) - return + if f.Header().Flags.Has(http2.FlagDataPadded) { + if w := s.fc.onRead(size - uint32(len(f.Data()))); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{s.id, w}) + } } - s.mu.Unlock() // TODO(bradfitz, zhaoq): A copy is required here because there is no // guarantee f.Data() is consumed before the arrival of next frame. // Can this copy be eliminated? - data := make([]byte, size) - copy(data, f.Data()) - s.write(recvMsg{data: data}) + if len(f.Data()) > 0 { + data := make([]byte, len(f.Data())) + copy(data, f.Data()) + s.write(recvMsg{data: data}) + } } if f.Header().Flags.Has(http2.FlagDataEndStream) { // Received the end of stream from the client. - s.mu.Lock() - if s.state != streamDone { - s.state = streamReadDone - } - s.mu.Unlock() + s.compareAndSwapState(streamActive, streamReadDone) s.write(recvMsg{err: io.EOF}) } } @@ -432,7 +603,7 @@ func (t *http2Server) handleRSTStream(f *http2.RSTStreamFrame) { if !ok { return } - t.closeStream(s) + t.closeStream(s, false, 0, nil, false) } func (t *http2Server) handleSettings(f *http2.SettingsFrame) { @@ -444,68 +615,76 @@ func (t *http2Server) handleSettings(f *http2.SettingsFrame) { ss = append(ss, s) return nil }) - // The settings will be applied once the ack is sent. - t.controlBuf.put(&settings{ack: true, ss: ss}) + t.controlBuf.put(&incomingSettings{ + ss: ss, + }) } +const ( + maxPingStrikes = 2 + defaultPingTimeout = 2 * time.Hour +) + func (t *http2Server) handlePing(f *http2.PingFrame) { - if f.IsAck() { // Do nothing. + if f.IsAck() { + if f.Data == goAwayPing.data && t.drainChan != nil { + close(t.drainChan) + return + } + // Maybe it's a BDP ping. + if t.bdpEst != nil { + t.bdpEst.calculate(f.Data) + } return } pingAck := &ping{ack: true} copy(pingAck.data[:], f.Data[:]) t.controlBuf.put(pingAck) -} -func (t *http2Server) handleWindowUpdate(f *http2.WindowUpdateFrame) { - id := f.Header().StreamID - incr := f.Increment - if id == 0 { - t.sendQuotaPool.add(int(incr)) + now := time.Now() + defer func() { + t.lastPingAt = now + }() + // A reset ping strikes means that we don't need to check for policy + // violation for this ping and the pingStrikes counter should be set + // to 0. + if atomic.CompareAndSwapUint32(&t.resetPingStrikes, 1, 0) { + t.pingStrikes = 0 return } - if s, ok := t.getStream(f); ok { - s.sendQuotaPool.add(int(incr)) - } -} - -func (t *http2Server) writeHeaders(s *Stream, b *bytes.Buffer, endStream bool) error { - first := true - endHeaders := false - var err error - // Sends the headers in a single batch. - for !endHeaders { - size := t.hBuf.Len() - if size > http2MaxFrameLen { - size = http2MaxFrameLen - } else { - endHeaders = true - } - if first { - p := http2.HeadersFrameParam{ - StreamID: s.id, - BlockFragment: b.Next(size), - EndStream: endStream, - EndHeaders: endHeaders, - } - err = t.framer.writeHeaders(endHeaders, p) - first = false - } else { - err = t.framer.writeContinuation(endHeaders, s.id, endHeaders, b.Next(size)) + t.mu.Lock() + ns := len(t.activeStreams) + t.mu.Unlock() + if ns < 1 && !t.kep.PermitWithoutStream { + // Keepalive shouldn't be active thus, this new ping should + // have come after at least defaultPingTimeout. + if t.lastPingAt.Add(defaultPingTimeout).After(now) { + t.pingStrikes++ } - if err != nil { - t.Close() - return connectionErrorf(true, err, "transport: %v", err) + } else { + // Check if keepalive policy is respected. + if t.lastPingAt.Add(t.kep.MinTime).After(now) { + t.pingStrikes++ } } - return nil + + if t.pingStrikes > maxPingStrikes { + // Send goaway and close the connection. + errorf("transport: Got too many pings from the client, closing the connection.") + t.controlBuf.put(&goAway{code: http2.ErrCodeEnhanceYourCalm, debugData: []byte("too_many_pings"), closeConn: true}) + } +} + +func (t *http2Server) handleWindowUpdate(f *http2.WindowUpdateFrame) { + t.controlBuf.put(&incomingWindowUpdate{ + streamID: f.Header().StreamID, + increment: f.Increment, + }) } // WriteHeader sends the header metedata md back to the client. func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { - s.mu.Lock() - if s.headerOk || s.state == streamDone { - s.mu.Unlock() + if s.headerOk || s.getState() == streamDone { return ErrIllegalHeaderWrite } s.headerOk = true @@ -517,36 +696,38 @@ func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { } } md = s.header - s.mu.Unlock() - if _, err := wait(s.ctx, nil, nil, t.shutdownChan, t.writableChan); err != nil { - return err - } - t.hBuf.Reset() - t.hEnc.WriteField(hpack.HeaderField{Name: ":status", Value: "200"}) - t.hEnc.WriteField(hpack.HeaderField{Name: "content-type", Value: "application/grpc"}) + // TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields + // first and create a slice of that exact size. + headerFields := make([]hpack.HeaderField, 0, 2) // at least :status, content-type will be there if none else. + headerFields = append(headerFields, hpack.HeaderField{Name: ":status", Value: "200"}) + headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: contentType(s.contentSubtype)}) if s.sendCompress != "" { - t.hEnc.WriteField(hpack.HeaderField{Name: "grpc-encoding", Value: s.sendCompress}) + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-encoding", Value: s.sendCompress}) } - for k, v := range md { + for k, vv := range md { if isReservedHeader(k) { // Clients don't tolerate reading restricted headers after some non restricted ones were sent. continue } - for _, entry := range v { - t.hEnc.WriteField(hpack.HeaderField{Name: k, Value: entry}) + for _, v := range vv { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) } } - bufLen := t.hBuf.Len() - if err := t.writeHeaders(s, t.hBuf, false); err != nil { - return err - } - if stats.On() { - outHeader := &stats.OutHeader{ - WireLength: bufLen, - } - stats.HandleRPC(s.Context(), outHeader) + t.controlBuf.put(&headerFrame{ + streamID: s.id, + hf: headerFields, + endStream: false, + onWrite: func() { + atomic.StoreUint32(&t.resetPingStrikes, 1) + }, + wq: s.wq, + }) + if t.stats != nil { + // Note: WireLength is not set in outHeader. + // TODO(mmukhi): Revisit this later, if needed. + outHeader := &stats.OutHeader{} + t.stats.HandleRPC(s.Context(), outHeader) } - t.writableChan <- 0 return nil } @@ -554,217 +735,188 @@ func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { // There is no further I/O operations being able to perform on this stream. // TODO(zhaoq): Now it indicates the end of entire stream. Revisit if early // OK is adopted. -func (t *http2Server) WriteStatus(s *Stream, statusCode codes.Code, statusDesc string) error { - var headersSent, hasHeader bool - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - return nil - } - if s.headerOk { - headersSent = true +func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { + if !s.headerOk && s.header.Len() > 0 { + if err := t.WriteHeader(s, nil); err != nil { + return err + } + } else { + if s.getState() == streamDone { + return nil + } } - if s.header.Len() > 0 { - hasHeader = true + // TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields + // first and create a slice of that exact size. + headerFields := make([]hpack.HeaderField, 0, 2) // grpc-status and grpc-message will be there if none else. + if !s.headerOk { + headerFields = append(headerFields, hpack.HeaderField{Name: ":status", Value: "200"}) + headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: contentType(s.contentSubtype)}) } - s.mu.Unlock() + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status", Value: strconv.Itoa(int(st.Code()))}) + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-message", Value: encodeGrpcMessage(st.Message())}) - if !headersSent && hasHeader { - t.WriteHeader(s, nil) - headersSent = true - } + if p := st.Proto(); p != nil && len(p.Details) > 0 { + stBytes, err := proto.Marshal(p) + if err != nil { + // TODO: return error instead, when callers are able to handle it. + panic(err) + } - if _, err := wait(s.ctx, nil, nil, t.shutdownChan, t.writableChan); err != nil { - return err - } - t.hBuf.Reset() - if !headersSent { - t.hEnc.WriteField(hpack.HeaderField{Name: ":status", Value: "200"}) - t.hEnc.WriteField(hpack.HeaderField{Name: "content-type", Value: "application/grpc"}) + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status-details-bin", Value: encodeBinHeader(stBytes)}) } - t.hEnc.WriteField( - hpack.HeaderField{ - Name: "grpc-status", - Value: strconv.Itoa(int(statusCode)), - }) - t.hEnc.WriteField(hpack.HeaderField{Name: "grpc-message", Value: encodeGrpcMessage(statusDesc)}) + // Attach the trailer metadata. - for k, v := range s.trailer { + for k, vv := range s.trailer { // Clients don't tolerate reading restricted headers after some non restricted ones were sent. if isReservedHeader(k) { continue } - for _, entry := range v { - t.hEnc.WriteField(hpack.HeaderField{Name: k, Value: entry}) + for _, v := range vv { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) } } - bufLen := t.hBuf.Len() - if err := t.writeHeaders(s, t.hBuf, true); err != nil { - t.Close() - return err + trailer := &headerFrame{ + streamID: s.id, + hf: headerFields, + endStream: true, + onWrite: func() { + atomic.StoreUint32(&t.resetPingStrikes, 1) + }, } - if stats.On() { - outTrailer := &stats.OutTrailer{ - WireLength: bufLen, - } - stats.HandleRPC(s.Context(), outTrailer) + t.closeStream(s, false, 0, trailer, true) + if t.stats != nil { + t.stats.HandleRPC(s.Context(), &stats.OutTrailer{}) } - t.closeStream(s) - t.writableChan <- 0 return nil } // Write converts the data into HTTP2 data frame and sends it out. Non-nil error // is returns if it fails (e.g., framing error, transport error). -func (t *http2Server) Write(s *Stream, data []byte, opts *Options) error { - // TODO(zhaoq): Support multi-writers for a single stream. - var writeHeaderFrame bool - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - return streamErrorf(codes.Unknown, "the stream has been done") - } - if !s.headerOk { - writeHeaderFrame = true - } - s.mu.Unlock() - if writeHeaderFrame { - t.WriteHeader(s, nil) - } - r := bytes.NewBuffer(data) - for { - if r.Len() == 0 { - return nil +func (t *http2Server) Write(s *Stream, hdr []byte, data []byte, opts *Options) error { + if !s.headerOk { // Headers haven't been written yet. + if err := t.WriteHeader(s, nil); err != nil { + // TODO(mmukhi, dfawley): Make sure this is the right code to return. + return streamErrorf(codes.Internal, "transport: %v", err) } - size := http2MaxFrameLen - // Wait until the stream has some quota to send the data. - sq, err := wait(s.ctx, nil, nil, t.shutdownChan, s.sendQuotaPool.acquire()) - if err != nil { - return err - } - // Wait until the transport has some quota to send the data. - tq, err := wait(s.ctx, nil, nil, t.shutdownChan, t.sendQuotaPool.acquire()) - if err != nil { - return err - } - if sq < size { - size = sq - } - if tq < size { - size = tq - } - p := r.Next(size) - ps := len(p) - if ps < sq { - // Overbooked stream quota. Return it back. - s.sendQuotaPool.add(sq - ps) - } - if ps < tq { - // Overbooked transport quota. Return it back. - t.sendQuotaPool.add(tq - ps) - } - t.framer.adjustNumWriters(1) - // Got some quota. Try to acquire writing privilege on the - // transport. - if _, err := wait(s.ctx, nil, nil, t.shutdownChan, t.writableChan); err != nil { - if _, ok := err.(StreamError); ok { - // Return the connection quota back. - t.sendQuotaPool.add(ps) - } - if t.framer.adjustNumWriters(-1) == 0 { - // This writer is the last one in this batch and has the - // responsibility to flush the buffered frames. It queues - // a flush request to controlBuf instead of flushing directly - // in order to avoid the race with other writing or flushing. - t.controlBuf.put(&flushIO{}) + } else { + // Writing headers checks for this condition. + if s.getState() == streamDone { + // TODO(mmukhi, dfawley): Should the server write also return io.EOF? + s.cancel() + select { + case <-t.ctx.Done(): + return ErrConnClosing + default: } - return err + return ContextErr(s.ctx.Err()) } + } + // Add some data to header frame so that we can equally distribute bytes across frames. + emptyLen := http2MaxFrameLen - len(hdr) + if emptyLen > len(data) { + emptyLen = len(data) + } + hdr = append(hdr, data[:emptyLen]...) + data = data[emptyLen:] + df := &dataFrame{ + streamID: s.id, + h: hdr, + d: data, + onEachWrite: func() { + atomic.StoreUint32(&t.resetPingStrikes, 1) + }, + } + if err := s.wq.get(int32(len(hdr) + len(data))); err != nil { select { - case <-s.ctx.Done(): - t.sendQuotaPool.add(ps) - if t.framer.adjustNumWriters(-1) == 0 { - t.controlBuf.put(&flushIO{}) - } - t.writableChan <- 0 - return ContextErr(s.ctx.Err()) + case <-t.ctx.Done(): + return ErrConnClosing default: } - var forceFlush bool - if r.Len() == 0 && t.framer.adjustNumWriters(0) == 1 && !opts.Last { - forceFlush = true - } - if err := t.framer.writeData(forceFlush, s.id, false, p); err != nil { - t.Close() - return connectionErrorf(true, err, "transport: %v", err) - } - if t.framer.adjustNumWriters(-1) == 0 { - t.framer.flushWrite() - } - t.writableChan <- 0 + return ContextErr(s.ctx.Err()) } - + return t.controlBuf.put(df) } -func (t *http2Server) applySettings(ss []http2.Setting) { - for _, s := range ss { - if s.ID == http2.SettingInitialWindowSize { - t.mu.Lock() - defer t.mu.Unlock() - for _, stream := range t.activeStreams { - stream.sendQuotaPool.add(int(s.Val - t.streamSendQuota)) - } - t.streamSendQuota = s.Val +// keepalive running in a separate goroutine does the following: +// 1. Gracefully closes an idle connection after a duration of keepalive.MaxConnectionIdle. +// 2. Gracefully closes any connection after a duration of keepalive.MaxConnectionAge. +// 3. Forcibly closes a connection after an additive period of keepalive.MaxConnectionAgeGrace over keepalive.MaxConnectionAge. +// 4. Makes sure a connection is alive by sending pings with a frequency of keepalive.Time and closes a non-responsive connection +// after an additional duration of keepalive.Timeout. +func (t *http2Server) keepalive() { + p := &ping{} + var pingSent bool + maxIdle := time.NewTimer(t.kp.MaxConnectionIdle) + maxAge := time.NewTimer(t.kp.MaxConnectionAge) + keepalive := time.NewTimer(t.kp.Time) + // NOTE: All exit paths of this function should reset their + // respective timers. A failure to do so will cause the + // following clean-up to deadlock and eventually leak. + defer func() { + if !maxIdle.Stop() { + <-maxIdle.C } - - } -} - -// controller running in a separate goroutine takes charge of sending control -// frames (e.g., window update, reset stream, setting, etc.) to the server. -func (t *http2Server) controller() { + if !maxAge.Stop() { + <-maxAge.C + } + if !keepalive.Stop() { + <-keepalive.C + } + }() for { select { - case i := <-t.controlBuf.get(): - t.controlBuf.load() + case <-maxIdle.C: + t.mu.Lock() + idle := t.idle + if idle.IsZero() { // The connection is non-idle. + t.mu.Unlock() + maxIdle.Reset(t.kp.MaxConnectionIdle) + continue + } + val := t.kp.MaxConnectionIdle - time.Since(idle) + t.mu.Unlock() + if val <= 0 { + // The connection has been idle for a duration of keepalive.MaxConnectionIdle or more. + // Gracefully close the connection. + t.drain(http2.ErrCodeNo, []byte{}) + // Resetting the timer so that the clean-up doesn't deadlock. + maxIdle.Reset(infinity) + return + } + maxIdle.Reset(val) + case <-maxAge.C: + t.drain(http2.ErrCodeNo, []byte{}) + maxAge.Reset(t.kp.MaxConnectionAgeGrace) select { - case <-t.writableChan: - switch i := i.(type) { - case *windowUpdate: - t.framer.writeWindowUpdate(true, i.streamID, i.increment) - case *settings: - if i.ack { - t.framer.writeSettingsAck(true) - t.applySettings(i.ss) - } else { - t.framer.writeSettings(true, i.ss...) - } - case *resetStream: - t.framer.writeRSTStream(true, i.streamID, i.code) - case *goAway: - t.mu.Lock() - if t.state == closing { - t.mu.Unlock() - // The transport is closing. - return - } - sid := t.maxStreamID - t.state = draining - t.mu.Unlock() - t.framer.writeGoAway(true, sid, http2.ErrCodeNo, nil) - case *flushIO: - t.framer.flushWrite() - case *ping: - t.framer.writePing(true, i.ack, i.data) - default: - grpclog.Printf("transport: http2Server.controller got unexpected item type %v\n", i) - } - t.writableChan <- 0 + case <-maxAge.C: + // Close the connection after grace period. + t.Close() + // Resetting the timer so that the clean-up doesn't deadlock. + maxAge.Reset(infinity) + case <-t.ctx.Done(): + } + return + case <-keepalive.C: + if atomic.CompareAndSwapUint32(&t.activity, 1, 0) { + pingSent = false + keepalive.Reset(t.kp.Time) continue - case <-t.shutdownChan: + } + if pingSent { + t.Close() + // Resetting the timer so that the clean-up doesn't deadlock. + keepalive.Reset(infinity) return } - case <-t.shutdownChan: + pingSent = true + if channelz.IsOn() { + t.czmu.Lock() + t.kpCount++ + t.czmu.Unlock() + } + t.controlBuf.put(p) + keepalive.Reset(t.kp.Timeout) + case <-t.ctx.Done(): return } } @@ -773,7 +925,7 @@ func (t *http2Server) controller() { // Close starts shutting down the http2Server transport. // TODO(zhaoq): Now the destruction is not blocked on any pending streams. This // could cause some resource issue. Revisit this later. -func (t *http2Server) Close() (err error) { +func (t *http2Server) Close() error { t.mu.Lock() if t.state == closing { t.mu.Unlock() @@ -783,44 +935,64 @@ func (t *http2Server) Close() (err error) { streams := t.activeStreams t.activeStreams = nil t.mu.Unlock() - close(t.shutdownChan) - err = t.conn.Close() + t.controlBuf.finish() + t.cancel() + err := t.conn.Close() + if channelz.IsOn() { + channelz.RemoveEntry(t.channelzID) + } // Cancel all active streams. for _, s := range streams { s.cancel() } - if stats.On() { + if t.stats != nil { connEnd := &stats.ConnEnd{} - stats.HandleConn(t.ctx, connEnd) + t.stats.HandleConn(t.ctx, connEnd) } - return + return err } // closeStream clears the footprint of a stream when the stream is not needed // any more. -func (t *http2Server) closeStream(s *Stream) { - t.mu.Lock() - delete(t.activeStreams, s.id) - if t.state == draining && len(t.activeStreams) == 0 { - defer t.Close() +func (t *http2Server) closeStream(s *Stream, rst bool, rstCode http2.ErrCode, hdr *headerFrame, eosReceived bool) { + if s.swapState(streamDone) == streamDone { + // If the stream was already done, return. + return } - t.mu.Unlock() // In case stream sending and receiving are invoked in separate // goroutines (e.g., bi-directional streaming), cancel needs to be // called to interrupt the potential blocking on other goroutines. s.cancel() - s.mu.Lock() - if q := s.fc.resetPendingData(); q > 0 { - if w := t.fc.onRead(q); w > 0 { - t.controlBuf.put(&windowUpdate{0, w}) - } + cleanup := &cleanupStream{ + streamID: s.id, + rst: rst, + rstCode: rstCode, + onWrite: func() { + t.mu.Lock() + if t.activeStreams != nil { + delete(t.activeStreams, s.id) + if len(t.activeStreams) == 0 { + t.idle = time.Now() + } + } + t.mu.Unlock() + if channelz.IsOn() { + t.czmu.Lock() + if eosReceived { + t.streamsSucceeded++ + } else { + t.streamsFailed++ + } + t.czmu.Unlock() + } + }, } - if s.state == streamDone { - s.mu.Unlock() - return + if hdr != nil { + hdr.cleanup = cleanup + t.controlBuf.put(hdr) + } else { + t.controlBuf.put(cleanup) } - s.state = streamDone - s.mu.Unlock() } func (t *http2Server) RemoteAddr() net.Addr { @@ -828,5 +1000,137 @@ func (t *http2Server) RemoteAddr() net.Addr { } func (t *http2Server) Drain() { - t.controlBuf.put(&goAway{}) + t.drain(http2.ErrCodeNo, []byte{}) +} + +func (t *http2Server) drain(code http2.ErrCode, debugData []byte) { + t.mu.Lock() + defer t.mu.Unlock() + if t.drainChan != nil { + return + } + t.drainChan = make(chan struct{}) + t.controlBuf.put(&goAway{code: code, debugData: debugData, headsUp: true}) +} + +var goAwayPing = &ping{data: [8]byte{1, 6, 1, 8, 0, 3, 3, 9}} + +// Handles outgoing GoAway and returns true if loopy needs to put itself +// in draining mode. +func (t *http2Server) outgoingGoAwayHandler(g *goAway) (bool, error) { + t.mu.Lock() + if t.state == closing { // TODO(mmukhi): This seems unnecessary. + t.mu.Unlock() + // The transport is closing. + return false, ErrConnClosing + } + sid := t.maxStreamID + if !g.headsUp { + // Stop accepting more streams now. + t.state = draining + if len(t.activeStreams) == 0 { + g.closeConn = true + } + t.mu.Unlock() + if err := t.framer.fr.WriteGoAway(sid, g.code, g.debugData); err != nil { + return false, err + } + if g.closeConn { + // Abruptly close the connection following the GoAway (via + // loopywriter). But flush out what's inside the buffer first. + t.framer.writer.Flush() + return false, fmt.Errorf("transport: Connection closing") + } + return true, nil + } + t.mu.Unlock() + // For a graceful close, send out a GoAway with stream ID of MaxUInt32, + // Follow that with a ping and wait for the ack to come back or a timer + // to expire. During this time accept new streams since they might have + // originated before the GoAway reaches the client. + // After getting the ack or timer expiration send out another GoAway this + // time with an ID of the max stream server intends to process. + if err := t.framer.fr.WriteGoAway(math.MaxUint32, http2.ErrCodeNo, []byte{}); err != nil { + return false, err + } + if err := t.framer.fr.WritePing(false, goAwayPing.data); err != nil { + return false, err + } + go func() { + timer := time.NewTimer(time.Minute) + defer timer.Stop() + select { + case <-t.drainChan: + case <-timer.C: + case <-t.ctx.Done(): + return + } + t.controlBuf.put(&goAway{code: g.code, debugData: g.debugData}) + }() + return false, nil +} + +func (t *http2Server) ChannelzMetric() *channelz.SocketInternalMetric { + t.czmu.RLock() + s := channelz.SocketInternalMetric{ + StreamsStarted: t.streamsStarted, + StreamsSucceeded: t.streamsSucceeded, + StreamsFailed: t.streamsFailed, + MessagesSent: t.msgSent, + MessagesReceived: t.msgRecv, + KeepAlivesSent: t.kpCount, + LastRemoteStreamCreatedTimestamp: t.lastStreamCreated, + LastMessageSentTimestamp: t.lastMsgSent, + LastMessageReceivedTimestamp: t.lastMsgRecv, + LocalFlowControlWindow: int64(t.fc.getSize()), + //socket options + LocalAddr: t.localAddr, + RemoteAddr: t.remoteAddr, + // Security + // RemoteName : + } + t.czmu.RUnlock() + s.RemoteFlowControlWindow = t.getOutFlowWindow() + return &s +} + +func (t *http2Server) IncrMsgSent() { + t.czmu.Lock() + t.msgSent++ + t.lastMsgSent = time.Now() + t.czmu.Unlock() +} + +func (t *http2Server) IncrMsgRecv() { + t.czmu.Lock() + t.msgRecv++ + t.lastMsgRecv = time.Now() + t.czmu.Unlock() +} + +func (t *http2Server) getOutFlowWindow() int64 { + resp := make(chan uint32) + timer := time.NewTimer(time.Second) + defer timer.Stop() + t.controlBuf.put(&outFlowControlSizeRequest{resp}) + select { + case sz := <-resp: + return int64(sz) + case <-t.ctxDone: + return -1 + case <-timer.C: + return -2 + } +} + +var rgen = rand.New(rand.NewSource(time.Now().UnixNano())) + +func getJitter(v time.Duration) time.Duration { + if v == infinity { + return 0 + } + // Generate a jitter between +/- 10% of the value. + r := int64(v / 10) + j := rgen.Int63n(2*r) - r + return time.Duration(j) } diff --git a/vendor/google.golang.org/grpc/transport/http_util.go b/vendor/google.golang.org/grpc/transport/http_util.go index a3c68d4cac7..a355866089c 100644 --- a/vendor/google.golang.org/grpc/transport/http_util.go +++ b/vendor/google.golang.org/grpc/transport/http_util.go @@ -1,33 +1,18 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ @@ -36,30 +21,36 @@ package transport import ( "bufio" "bytes" + "encoding/base64" "fmt" - "io" "net" + "net/http" "strconv" "strings" - "sync/atomic" "time" + "github.com/golang/protobuf/proto" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" + spb "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/grpc/codes" - "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" ) const ( - // The primary user agent - primaryUA = "grpc-go/1.0" // http2MaxFrameLen specifies the max length of a HTTP2 frame. http2MaxFrameLen = 16384 // 16KB frame // http://http2.github.io/http2-spec/#SettingValues http2InitHeaderTableSize = 4096 // http2IOBufSize specifies the buffer size for sending frames. - http2IOBufSize = 32 * 1024 + defaultWriteBufSize = 32 * 1024 + defaultReadBufSize = 32 * 1024 + // baseContentType is the base content-type for gRPC. This is a valid + // content-type on it's own, but can also include a content-subtype such as + // "proto" as a suffix after "+" or ";". See + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests + // for more details. + baseContentType = "application/grpc" ) var ( @@ -78,7 +69,7 @@ var ( http2.ErrCodeConnect: codes.Internal, http2.ErrCodeEnhanceYourCalm: codes.ResourceExhausted, http2.ErrCodeInadequateSecurity: codes.PermissionDenied, - http2.ErrCodeHTTP11Required: codes.FailedPrecondition, + http2.ErrCodeHTTP11Required: codes.Internal, } statusCodeConvTab = map[codes.Code]http2.ErrCode{ codes.Internal: http2.ErrCodeInternal, @@ -87,24 +78,48 @@ var ( codes.ResourceExhausted: http2.ErrCodeEnhanceYourCalm, codes.PermissionDenied: http2.ErrCodeInadequateSecurity, } + httpStatusConvTab = map[int]codes.Code{ + // 400 Bad Request - INTERNAL. + http.StatusBadRequest: codes.Internal, + // 401 Unauthorized - UNAUTHENTICATED. + http.StatusUnauthorized: codes.Unauthenticated, + // 403 Forbidden - PERMISSION_DENIED. + http.StatusForbidden: codes.PermissionDenied, + // 404 Not Found - UNIMPLEMENTED. + http.StatusNotFound: codes.Unimplemented, + // 429 Too Many Requests - UNAVAILABLE. + http.StatusTooManyRequests: codes.Unavailable, + // 502 Bad Gateway - UNAVAILABLE. + http.StatusBadGateway: codes.Unavailable, + // 503 Service Unavailable - UNAVAILABLE. + http.StatusServiceUnavailable: codes.Unavailable, + // 504 Gateway timeout - UNAVAILABLE. + http.StatusGatewayTimeout: codes.Unavailable, + } ) // Records the states during HPACK decoding. Must be reset once the // decoding of the entire headers are finished. type decodeState struct { - err error // first error encountered decoding - encoding string - // statusCode caches the stream status received from the trailer - // the server sent. Client side only. - statusCode codes.Code - statusDesc string + // statusGen caches the stream status received from the trailer the server + // sent. Client side only. Do not access directly. After all trailers are + // parsed, use the status method to retrieve the status. + statusGen *status.Status + // rawStatusCode and rawStatusMsg are set from the raw trailer fields and are not + // intended for direct access outside of parsing. + rawStatusCode *int + rawStatusMsg string + httpStatus *int // Server side only fields. timeoutSet bool timeout time.Duration method string // key-value metadata map from the peer. - mdata map[string][]string + mdata map[string][]string + statsTags []byte + statsTrace []byte + contentSubtype string } // isReservedHeader checks whether hdr belongs to HTTP2 headers @@ -116,11 +131,13 @@ func isReservedHeader(hdr string) bool { } switch hdr { case "content-type", + "user-agent", "grpc-message-type", "grpc-encoding", "grpc-message", "grpc-status", "grpc-timeout", + "grpc-status-details-bin", "te": return true default: @@ -128,86 +145,212 @@ func isReservedHeader(hdr string) bool { } } -// isWhitelistedPseudoHeader checks whether hdr belongs to HTTP2 pseudoheaders -// that should be propagated into metadata visible to users. -func isWhitelistedPseudoHeader(hdr string) bool { +// isWhitelistedHeader checks whether hdr should be propagated +// into metadata visible to users. +func isWhitelistedHeader(hdr string) bool { switch hdr { - case ":authority": + case ":authority", "user-agent": return true default: return false } } -func (d *decodeState) setErr(err error) { - if d.err == nil { - d.err = err +// contentSubtype returns the content-subtype for the given content-type. The +// given content-type must be a valid content-type that starts with +// "application/grpc". A content-subtype will follow "application/grpc" after a +// "+" or ";". See +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +// +// If contentType is not a valid content-type for gRPC, the boolean +// will be false, otherwise true. If content-type == "application/grpc", +// "application/grpc+", or "application/grpc;", the boolean will be true, +// but no content-subtype will be returned. +// +// contentType is assumed to be lowercase already. +func contentSubtype(contentType string) (string, bool) { + if contentType == baseContentType { + return "", true + } + if !strings.HasPrefix(contentType, baseContentType) { + return "", false + } + // guaranteed since != baseContentType and has baseContentType prefix + switch contentType[len(baseContentType)] { + case '+', ';': + // this will return true for "application/grpc+" or "application/grpc;" + // which the previous validContentType function tested to be valid, so we + // just say that no content-subtype is specified in this case + return contentType[len(baseContentType)+1:], true + default: + return "", false } } -func validContentType(t string) bool { - e := "application/grpc" - if !strings.HasPrefix(t, e) { - return false +// contentSubtype is assumed to be lowercase +func contentType(contentSubtype string) string { + if contentSubtype == "" { + return baseContentType } - // Support variations on the content-type - // (e.g. "application/grpc+blah", "application/grpc;blah"). - if len(t) > len(e) && t[len(e)] != '+' && t[len(e)] != ';' { - return false + return baseContentType + "+" + contentSubtype +} + +func (d *decodeState) status() *status.Status { + if d.statusGen == nil { + // No status-details were provided; generate status using code/msg. + d.statusGen = status.New(codes.Code(int32(*(d.rawStatusCode))), d.rawStatusMsg) + } + return d.statusGen +} + +const binHdrSuffix = "-bin" + +func encodeBinHeader(v []byte) string { + return base64.RawStdEncoding.EncodeToString(v) +} + +func decodeBinHeader(v string) ([]byte, error) { + if len(v)%4 == 0 { + // Input was padded, or padding was not necessary. + return base64.StdEncoding.DecodeString(v) + } + return base64.RawStdEncoding.DecodeString(v) +} + +func encodeMetadataHeader(k, v string) string { + if strings.HasSuffix(k, binHdrSuffix) { + return encodeBinHeader(([]byte)(v)) + } + return v +} + +func decodeMetadataHeader(k, v string) (string, error) { + if strings.HasSuffix(k, binHdrSuffix) { + b, err := decodeBinHeader(v) + return string(b), err + } + return v, nil +} + +func (d *decodeState) decodeResponseHeader(frame *http2.MetaHeadersFrame) error { + for _, hf := range frame.Fields { + if err := d.processHeaderField(hf); err != nil { + return err + } + } + + // If grpc status exists, no need to check further. + if d.rawStatusCode != nil || d.statusGen != nil { + return nil + } + + // If grpc status doesn't exist and http status doesn't exist, + // then it's a malformed header. + if d.httpStatus == nil { + return streamErrorf(codes.Internal, "malformed header: doesn't contain status(gRPC or HTTP)") + } + + if *(d.httpStatus) != http.StatusOK { + code, ok := httpStatusConvTab[*(d.httpStatus)] + if !ok { + code = codes.Unknown + } + return streamErrorf(code, http.StatusText(*(d.httpStatus))) + } + + // gRPC status doesn't exist and http status is OK. + // Set rawStatusCode to be unknown and return nil error. + // So that, if the stream has ended this Unknown status + // will be propagated to the user. + // Otherwise, it will be ignored. In which case, status from + // a later trailer, that has StreamEnded flag set, is propagated. + code := int(codes.Unknown) + d.rawStatusCode = &code + return nil + +} + +func (d *decodeState) addMetadata(k, v string) { + if d.mdata == nil { + d.mdata = make(map[string][]string) } - return true + d.mdata[k] = append(d.mdata[k], v) } -func (d *decodeState) processHeaderField(f hpack.HeaderField) { +func (d *decodeState) processHeaderField(f hpack.HeaderField) error { switch f.Name { case "content-type": - if !validContentType(f.Value) { - d.setErr(streamErrorf(codes.FailedPrecondition, "transport: received the unexpected content-type %q", f.Value)) - return + contentSubtype, validContentType := contentSubtype(f.Value) + if !validContentType { + return streamErrorf(codes.Internal, "transport: received the unexpected content-type %q", f.Value) } + d.contentSubtype = contentSubtype + // TODO: do we want to propagate the whole content-type in the metadata, + // or come up with a way to just propagate the content-subtype if it was set? + // ie {"content-type": "application/grpc+proto"} or {"content-subtype": "proto"} + // in the metadata? + d.addMetadata(f.Name, f.Value) case "grpc-encoding": d.encoding = f.Value case "grpc-status": code, err := strconv.Atoi(f.Value) if err != nil { - d.setErr(streamErrorf(codes.Internal, "transport: malformed grpc-status: %v", err)) - return + return streamErrorf(codes.Internal, "transport: malformed grpc-status: %v", err) } - d.statusCode = codes.Code(code) + d.rawStatusCode = &code case "grpc-message": - d.statusDesc = decodeGrpcMessage(f.Value) + d.rawStatusMsg = decodeGrpcMessage(f.Value) + case "grpc-status-details-bin": + v, err := decodeBinHeader(f.Value) + if err != nil { + return streamErrorf(codes.Internal, "transport: malformed grpc-status-details-bin: %v", err) + } + s := &spb.Status{} + if err := proto.Unmarshal(v, s); err != nil { + return streamErrorf(codes.Internal, "transport: malformed grpc-status-details-bin: %v", err) + } + d.statusGen = status.FromProto(s) case "grpc-timeout": d.timeoutSet = true var err error - d.timeout, err = decodeTimeout(f.Value) - if err != nil { - d.setErr(streamErrorf(codes.Internal, "transport: malformed time-out: %v", err)) - return + if d.timeout, err = decodeTimeout(f.Value); err != nil { + return streamErrorf(codes.Internal, "transport: malformed time-out: %v", err) } case ":path": d.method = f.Value + case ":status": + code, err := strconv.Atoi(f.Value) + if err != nil { + return streamErrorf(codes.Internal, "transport: malformed http-status: %v", err) + } + d.httpStatus = &code + case "grpc-tags-bin": + v, err := decodeBinHeader(f.Value) + if err != nil { + return streamErrorf(codes.Internal, "transport: malformed grpc-tags-bin: %v", err) + } + d.statsTags = v + d.addMetadata(f.Name, string(v)) + case "grpc-trace-bin": + v, err := decodeBinHeader(f.Value) + if err != nil { + return streamErrorf(codes.Internal, "transport: malformed grpc-trace-bin: %v", err) + } + d.statsTrace = v + d.addMetadata(f.Name, string(v)) default: - if !isReservedHeader(f.Name) || isWhitelistedPseudoHeader(f.Name) { - if f.Name == "user-agent" { - i := strings.LastIndex(f.Value, " ") - if i == -1 { - // There is no application user agent string being set. - return - } - // Extract the application user agent string. - f.Value = f.Value[:i] - } - if d.mdata == nil { - d.mdata = make(map[string][]string) - } - k, v, err := metadata.DecodeKeyValue(f.Name, f.Value) - if err != nil { - grpclog.Printf("Failed to decode (%q, %q): %v", f.Name, f.Value, err) - return - } - d.mdata[k] = append(d.mdata[k], v) + if isReservedHeader(f.Name) && !isWhitelistedHeader(f.Name) { + break + } + v, err := decodeMetadataHeader(f.Name, f.Value) + if err != nil { + errorf("Failed to decode metadata header (%q, %q): %v", f.Name, f.Value, err) + return nil } + d.addMetadata(f.Name, v) } + return nil } type timeoutUnit uint8 @@ -366,148 +509,66 @@ func decodeGrpcMessageUnchecked(msg string) string { return buf.String() } -type framer struct { - numWriters int32 - reader io.Reader - writer *bufio.Writer - fr *http2.Framer -} +type bufWriter struct { + buf []byte + offset int + batchSize int + conn net.Conn + err error -func newFramer(conn net.Conn) *framer { - f := &framer{ - reader: bufio.NewReaderSize(conn, http2IOBufSize), - writer: bufio.NewWriterSize(conn, http2IOBufSize), - } - f.fr = http2.NewFramer(f.writer, f.reader) - f.fr.ReadMetaHeaders = hpack.NewDecoder(http2InitHeaderTableSize, nil) - return f -} - -func (f *framer) adjustNumWriters(i int32) int32 { - return atomic.AddInt32(&f.numWriters, i) -} - -// The following writeXXX functions can only be called when the caller gets -// unblocked from writableChan channel (i.e., owns the privilege to write). - -func (f *framer) writeContinuation(forceFlush bool, streamID uint32, endHeaders bool, headerBlockFragment []byte) error { - if err := f.fr.WriteContinuation(streamID, endHeaders, headerBlockFragment); err != nil { - return err - } - if forceFlush { - return f.writer.Flush() - } - return nil -} - -func (f *framer) writeData(forceFlush bool, streamID uint32, endStream bool, data []byte) error { - if err := f.fr.WriteData(streamID, endStream, data); err != nil { - return err - } - if forceFlush { - return f.writer.Flush() - } - return nil + onFlush func() } -func (f *framer) writeGoAway(forceFlush bool, maxStreamID uint32, code http2.ErrCode, debugData []byte) error { - if err := f.fr.WriteGoAway(maxStreamID, code, debugData); err != nil { - return err - } - if forceFlush { - return f.writer.Flush() +func newBufWriter(conn net.Conn, batchSize int) *bufWriter { + return &bufWriter{ + buf: make([]byte, batchSize*2), + batchSize: batchSize, + conn: conn, } - return nil } -func (f *framer) writeHeaders(forceFlush bool, p http2.HeadersFrameParam) error { - if err := f.fr.WriteHeaders(p); err != nil { - return err +func (w *bufWriter) Write(b []byte) (n int, err error) { + if w.err != nil { + return 0, w.err } - if forceFlush { - return f.writer.Flush() + n = copy(w.buf[w.offset:], b) + w.offset += n + if w.offset >= w.batchSize { + err = w.Flush() } - return nil + return n, err } -func (f *framer) writePing(forceFlush, ack bool, data [8]byte) error { - if err := f.fr.WritePing(ack, data); err != nil { - return err - } - if forceFlush { - return f.writer.Flush() +func (w *bufWriter) Flush() error { + if w.err != nil { + return w.err } - return nil -} - -func (f *framer) writePriority(forceFlush bool, streamID uint32, p http2.PriorityParam) error { - if err := f.fr.WritePriority(streamID, p); err != nil { - return err + if w.offset == 0 { + return nil } - if forceFlush { - return f.writer.Flush() + if w.onFlush != nil { + w.onFlush() } - return nil + _, w.err = w.conn.Write(w.buf[:w.offset]) + w.offset = 0 + return w.err } -func (f *framer) writePushPromise(forceFlush bool, p http2.PushPromiseParam) error { - if err := f.fr.WritePushPromise(p); err != nil { - return err - } - if forceFlush { - return f.writer.Flush() - } - return nil -} - -func (f *framer) writeRSTStream(forceFlush bool, streamID uint32, code http2.ErrCode) error { - if err := f.fr.WriteRSTStream(streamID, code); err != nil { - return err - } - if forceFlush { - return f.writer.Flush() - } - return nil -} - -func (f *framer) writeSettings(forceFlush bool, settings ...http2.Setting) error { - if err := f.fr.WriteSettings(settings...); err != nil { - return err - } - if forceFlush { - return f.writer.Flush() - } - return nil -} - -func (f *framer) writeSettingsAck(forceFlush bool) error { - if err := f.fr.WriteSettingsAck(); err != nil { - return err - } - if forceFlush { - return f.writer.Flush() - } - return nil +type framer struct { + writer *bufWriter + fr *http2.Framer } -func (f *framer) writeWindowUpdate(forceFlush bool, streamID, incr uint32) error { - if err := f.fr.WriteWindowUpdate(streamID, incr); err != nil { - return err - } - if forceFlush { - return f.writer.Flush() +func newFramer(conn net.Conn, writeBufferSize, readBufferSize int) *framer { + r := bufio.NewReaderSize(conn, readBufferSize) + w := newBufWriter(conn, writeBufferSize) + f := &framer{ + writer: w, + fr: http2.NewFramer(w, r), } - return nil -} - -func (f *framer) flushWrite() error { - return f.writer.Flush() -} - -func (f *framer) readFrame() (http2.Frame, error) { - return f.fr.ReadFrame() -} - -func (f *framer) errorDetail() error { - return f.fr.ErrorDetail() + // Opt-in to Frame reuse API on framer to reduce garbage. + // Frames aren't safe to read from after a subsequent call to ReadFrame. + f.fr.SetReuseFrames() + f.fr.ReadMetaHeaders = hpack.NewDecoder(http2InitHeaderTableSize, nil) + return f } diff --git a/vendor/google.golang.org/grpc/transport/log.go b/vendor/google.golang.org/grpc/transport/log.go new file mode 100644 index 00000000000..ac8e358c5c8 --- /dev/null +++ b/vendor/google.golang.org/grpc/transport/log.go @@ -0,0 +1,50 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This file contains wrappers for grpclog functions. +// The transport package only logs to verbose level 2 by default. + +package transport + +import "google.golang.org/grpc/grpclog" + +const logLevel = 2 + +func infof(format string, args ...interface{}) { + if grpclog.V(logLevel) { + grpclog.Infof(format, args...) + } +} + +func warningf(format string, args ...interface{}) { + if grpclog.V(logLevel) { + grpclog.Warningf(format, args...) + } +} + +func errorf(format string, args ...interface{}) { + if grpclog.V(logLevel) { + grpclog.Errorf(format, args...) + } +} + +func fatalf(format string, args ...interface{}) { + if grpclog.V(logLevel) { + grpclog.Fatalf(format, args...) + } +} diff --git a/vendor/google.golang.org/grpc/transport/pre_go16.go b/vendor/google.golang.org/grpc/transport/pre_go16.go deleted file mode 100644 index 33d91c17c4e..00000000000 --- a/vendor/google.golang.org/grpc/transport/pre_go16.go +++ /dev/null @@ -1,51 +0,0 @@ -// +build !go1.6 - -/* - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -package transport - -import ( - "net" - "time" - - "golang.org/x/net/context" -) - -// dialContext connects to the address on the named network. -func dialContext(ctx context.Context, network, address string) (net.Conn, error) { - var dialer net.Dialer - if deadline, ok := ctx.Deadline(); ok { - dialer.Timeout = deadline.Sub(time.Now()) - } - return dialer.Dial(network, address) -} diff --git a/vendor/google.golang.org/grpc/transport/transport.go b/vendor/google.golang.org/grpc/transport/transport.go index 4726bb2cea6..2f643a3d042 100644 --- a/vendor/google.golang.org/grpc/transport/transport.go +++ b/vendor/google.golang.org/grpc/transport/transport.go @@ -1,53 +1,41 @@ /* * - * Copyright 2014, Google Inc. - * All rights reserved. + * Copyright 2014 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ -/* -Package transport defines and implements message oriented communication channel -to complete various transactions (e.g., an RPC). -*/ -package transport // import "google.golang.org/grpc/transport" +// Package transport defines and implements message oriented communication +// channel to complete various transactions (e.g., an RPC). It is meant for +// grpc-internal usage and is not intended to be imported directly by users. +package transport // externally used as import "google.golang.org/grpc/transport" import ( - "bytes" + "errors" "fmt" "io" "net" "sync" + "sync/atomic" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" "google.golang.org/grpc/tap" ) @@ -61,68 +49,76 @@ type recvMsg struct { err error } -func (*recvMsg) item() {} - -// All items in an out of a recvBuffer should be the same type. -type item interface { - item() -} - -// recvBuffer is an unbounded channel of item. +// recvBuffer is an unbounded channel of recvMsg structs. +// Note recvBuffer differs from controlBuffer only in that recvBuffer +// holds a channel of only recvMsg structs instead of objects implementing "item" interface. +// recvBuffer is written to much more often than +// controlBuffer and using strict recvMsg structs helps avoid allocation in "recvBuffer.put" type recvBuffer struct { - c chan item + c chan recvMsg mu sync.Mutex - backlog []item + backlog []recvMsg + err error } func newRecvBuffer() *recvBuffer { b := &recvBuffer{ - c: make(chan item, 1), + c: make(chan recvMsg, 1), } return b } -func (b *recvBuffer) put(r item) { +func (b *recvBuffer) put(r recvMsg) { b.mu.Lock() - defer b.mu.Unlock() + if b.err != nil { + b.mu.Unlock() + // An error had occurred earlier, don't accept more + // data or errors. + return + } + b.err = r.err if len(b.backlog) == 0 { select { case b.c <- r: + b.mu.Unlock() return default: } } b.backlog = append(b.backlog, r) + b.mu.Unlock() } func (b *recvBuffer) load() { b.mu.Lock() - defer b.mu.Unlock() if len(b.backlog) > 0 { select { case b.c <- b.backlog[0]: + b.backlog[0] = recvMsg{} b.backlog = b.backlog[1:] default: } } + b.mu.Unlock() } -// get returns the channel that receives an item in the buffer. +// get returns the channel that receives a recvMsg in the buffer. // -// Upon receipt of an item, the caller should call load to send another -// item onto the channel if there is any. -func (b *recvBuffer) get() <-chan item { +// Upon receipt of a recvMsg, the caller should call load to send another +// recvMsg onto the channel if there is any. +func (b *recvBuffer) get() <-chan recvMsg { return b.c } +// // recvBufferReader implements io.Reader interface to read the data from // recvBuffer. type recvBufferReader struct { - ctx context.Context - goAway chan struct{} - recv *recvBuffer - last *bytes.Reader // Stores the remaining data in the previous calls. - err error + ctx context.Context + ctxDone <-chan struct{} // cache of ctx.Done() (for performance). + recv *recvBuffer + last []byte // Stores the remaining data in the previous calls. + err error } // Read reads the next len(p) bytes from last. If last is drained, it tries to @@ -132,28 +128,32 @@ func (r *recvBufferReader) Read(p []byte) (n int, err error) { if r.err != nil { return 0, r.err } - defer func() { r.err = err }() - if r.last != nil && r.last.Len() > 0 { + n, r.err = r.read(p) + return n, r.err +} + +func (r *recvBufferReader) read(p []byte) (n int, err error) { + if r.last != nil && len(r.last) > 0 { // Read remaining data left in last call. - return r.last.Read(p) + copied := copy(p, r.last) + r.last = r.last[copied:] + return copied, nil } select { - case <-r.ctx.Done(): + case <-r.ctxDone: return 0, ContextErr(r.ctx.Err()) - case <-r.goAway: - return 0, ErrStreamDrain - case i := <-r.recv.get(): + case m := <-r.recv.get(): r.recv.load() - m := i.(*recvMsg) if m.err != nil { return 0, m.err } - r.last = bytes.NewReader(m.data) - return r.last.Read(p) + copied := copy(p, m.data) + r.last = m.data[copied:] + return copied, nil } } -type streamState uint8 +type streamState uint32 const ( streamActive streamState = iota @@ -164,59 +164,75 @@ const ( // Stream represents an RPC in the transport layer. type Stream struct { - id uint32 - // nil for client side Stream. - st ServerTransport - // clientStatsCtx keeps the user context for stats handling. - // It's only valid on client side. Server side stats context is same as s.ctx. - // All client side stats collection should use the clientStatsCtx (instead of the stream context) - // so that all the generated stats for a particular RPC can be associated in the processing phase. - clientStatsCtx context.Context - // ctx is the associated context of the stream. - ctx context.Context - // cancel is always nil for client side Stream. - cancel context.CancelFunc - // done is closed when the final status arrives. - done chan struct{} - // goAway is closed when the server sent GoAways signal before this stream was initiated. - goAway chan struct{} - // method records the associated RPC method of the stream. - method string + id uint32 + st ServerTransport // nil for client side Stream + ctx context.Context // the associated context of the stream + cancel context.CancelFunc // always nil for client side Stream + done chan struct{} // closed at the end of stream to unblock writers. On the client side. + ctxDone <-chan struct{} // same as done chan but for server side. Cache of ctx.Done() (for performance) + method string // the associated RPC method of the stream recvCompress string sendCompress string buf *recvBuffer - dec io.Reader + trReader io.Reader fc *inFlow recvQuota uint32 - // The accumulated inbound quota pending for window update. - updateQuota uint32 - // The handler to control the window update procedure for both this - // particular stream and the associated transport. - windowHandler func(int) + wq *writeQuota + + // Callback to state application's intentions to read data. This + // is used to adjust flow control, if needed. + requestRead func(int) - sendQuotaPool *quotaPool - // Close headerChan to indicate the end of reception of header metadata. - headerChan chan struct{} - // header caches the received header metadata. - header metadata.MD - // The key-value map of trailer metadata. - trailer metadata.MD - - mu sync.RWMutex // guard the following - // headerOK becomes true from the first header is about to send. - headerOk bool + headerChan chan struct{} // closed to indicate the end of header metadata. + headerDone uint32 // set when headerChan is closed. Used to avoid closing headerChan multiple times. + header metadata.MD // the received header metadata. + trailer metadata.MD // the key-value map of trailer metadata. + + headerOk bool // becomes true from the first header is about to send state streamState - // true iff headerChan is closed. Used to avoid closing headerChan - // multiple times. - headerDone bool - // the status received from the server. - statusCode codes.Code - statusDesc string + + status *status.Status // the status error received from the server + + bytesReceived uint32 // indicates whether any bytes have been received on this stream + unprocessed uint32 // set if the server sends a refused stream or GOAWAY including this stream + + // contentSubtype is the content-subtype for requests. + // this must be lowercase or the behavior is undefined. + contentSubtype string +} + +func (s *Stream) swapState(st streamState) streamState { + return streamState(atomic.SwapUint32((*uint32)(&s.state), uint32(st))) +} + +func (s *Stream) compareAndSwapState(oldState, newState streamState) bool { + return atomic.CompareAndSwapUint32((*uint32)(&s.state), uint32(oldState), uint32(newState)) +} + +func (s *Stream) getState() streamState { + return streamState(atomic.LoadUint32((*uint32)(&s.state))) +} + +func (s *Stream) waitOnHeader() error { + if s.headerChan == nil { + // On the server headerChan is always nil since a stream originates + // only after having received headers. + return nil + } + select { + case <-s.ctx.Done(): + return ContextErr(s.ctx.Err()) + case <-s.headerChan: + return nil + } } // RecvCompress returns the compression algorithm applied to the inbound // message. It is empty string if there is no compression applied. func (s *Stream) RecvCompress() string { + if err := s.waitOnHeader(); err != nil { + return "" + } return s.recvCompress } @@ -231,33 +247,31 @@ func (s *Stream) Done() <-chan struct{} { return s.done } -// GoAway returns a channel which is closed when the server sent GoAways signal -// before this stream was initiated. -func (s *Stream) GoAway() <-chan struct{} { - return s.goAway -} - // Header acquires the key-value pairs of header metadata once it // is available. It blocks until i) the metadata is ready or ii) there is no -// header metadata or iii) the stream is cancelled/expired. +// header metadata or iii) the stream is canceled/expired. func (s *Stream) Header() (metadata.MD, error) { + err := s.waitOnHeader() + // Even if the stream is closed, header is returned if available. select { - case <-s.ctx.Done(): - return nil, ContextErr(s.ctx.Err()) - case <-s.goAway: - return nil, ErrStreamDrain case <-s.headerChan: + if s.header == nil { + return nil, nil + } return s.header.Copy(), nil + default: } + return nil, err } // Trailer returns the cached trailer metedata. Note that if it is not called // after the entire stream is done, it could return an empty MD. Client // side only. +// It can be safely read only after stream has ended that is either read +// or write have returned io.EOF. func (s *Stream) Trailer() metadata.MD { - s.mu.RLock() - defer s.mu.RUnlock() - return s.trailer.Copy() + c := s.trailer.Copy() + return c } // ServerTransport returns the underlying ServerTransport for the stream. @@ -266,6 +280,15 @@ func (s *Stream) ServerTransport() ServerTransport { return s.st } +// ContentSubtype returns the content-subtype for a request. For example, a +// content-subtype of "proto" will result in a content-type of +// "application/grpc+proto". This will always be lowercase. See +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +func (s *Stream) ContentSubtype() string { + return s.contentSubtype +} + // Context returns the context of the stream. func (s *Stream) Context() context.Context { return s.ctx @@ -276,73 +299,97 @@ func (s *Stream) Method() string { return s.method } -// StatusCode returns statusCode received from the server. -func (s *Stream) StatusCode() codes.Code { - return s.statusCode -} - -// StatusDesc returns statusDesc received from the server. -func (s *Stream) StatusDesc() string { - return s.statusDesc +// Status returns the status received from the server. +// Status can be read safely only after the stream has ended, +// that is, read or write has returned io.EOF. +func (s *Stream) Status() *status.Status { + return s.status } // SetHeader sets the header metadata. This can be called multiple times. // Server side only. +// This should not be called in parallel to other data writes. func (s *Stream) SetHeader(md metadata.MD) error { - s.mu.Lock() - defer s.mu.Unlock() - if s.headerOk || s.state == streamDone { - return ErrIllegalHeaderWrite - } if md.Len() == 0 { return nil } + if s.headerOk || atomic.LoadUint32((*uint32)(&s.state)) == uint32(streamDone) { + return ErrIllegalHeaderWrite + } s.header = metadata.Join(s.header, md) return nil } +// SendHeader sends the given header metadata. The given metadata is +// combined with any metadata set by previous calls to SetHeader and +// then written to the transport stream. +func (s *Stream) SendHeader(md metadata.MD) error { + t := s.ServerTransport() + return t.WriteHeader(s, md) +} + // SetTrailer sets the trailer metadata which will be sent with the RPC status // by the server. This can be called multiple times. Server side only. +// This should not be called parallel to other data writes. func (s *Stream) SetTrailer(md metadata.MD) error { if md.Len() == 0 { return nil } - s.mu.Lock() - defer s.mu.Unlock() s.trailer = metadata.Join(s.trailer, md) return nil } func (s *Stream) write(m recvMsg) { - s.buf.put(&m) + s.buf.put(m) +} + +// Read reads all p bytes from the wire for this stream. +func (s *Stream) Read(p []byte) (n int, err error) { + // Don't request a read if there was an error earlier + if er := s.trReader.(*transportReader).er; er != nil { + return 0, er + } + s.requestRead(len(p)) + return io.ReadFull(s.trReader, p) } -// Read reads all the data available for this Stream from the transport and +// tranportReader reads all the data available for this Stream from the transport and // passes them into the decoder, which converts them into a gRPC message stream. // The error is io.EOF when the stream is done or another non-nil error if // the stream broke. -func (s *Stream) Read(p []byte) (n int, err error) { - n, err = s.dec.Read(p) +type transportReader struct { + reader io.Reader + // The handler to control the window update procedure for both this + // particular stream and the associated transport. + windowHandler func(int) + er error +} + +func (t *transportReader) Read(p []byte) (n int, err error) { + n, err = t.reader.Read(p) if err != nil { + t.er = err return } - s.windowHandler(n) + t.windowHandler(n) return } -// The key to save transport.Stream in the context. -type streamKey struct{} +// BytesReceived indicates whether any bytes have been received on this stream. +func (s *Stream) BytesReceived() bool { + return atomic.LoadUint32(&s.bytesReceived) == 1 +} -// newContextWithStream creates a new context from ctx and attaches stream -// to it. -func newContextWithStream(ctx context.Context, stream *Stream) context.Context { - return context.WithValue(ctx, streamKey{}, stream) +// Unprocessed indicates whether the server did not process this stream -- +// i.e. it sent a refused stream or GOAWAY including this stream ID. +func (s *Stream) Unprocessed() bool { + return atomic.LoadUint32(&s.unprocessed) == 1 } -// StreamFromContext returns the stream saved in ctx. -func StreamFromContext(ctx context.Context) (s *Stream, ok bool) { - s, ok = ctx.Value(streamKey{}).(*Stream) - return +// GoString is implemented by Stream so context.String() won't +// race when printing %#v. +func (s *Stream) GoString() string { + return fmt.Sprintf("", s, s.method) } // state of transport @@ -350,16 +397,23 @@ type transportState int const ( reachable transportState = iota - unreachable closing draining ) // ServerConfig consists of all the configurations to establish a server transport. type ServerConfig struct { - MaxStreams uint32 - AuthInfo credentials.AuthInfo - InTapHandle tap.ServerInHandle + MaxStreams uint32 + AuthInfo credentials.AuthInfo + InTapHandle tap.ServerInHandle + StatsHandler stats.Handler + KeepaliveParams keepalive.ServerParameters + KeepalivePolicy keepalive.EnforcementPolicy + InitialWindowSize int32 + InitialConnWindowSize int32 + WriteBufferSize int + ReadBufferSize int + ChannelzParentID int64 } // NewServerTransport creates a ServerTransport with conn or non-nil error @@ -372,6 +426,9 @@ func NewServerTransport(protocol string, conn net.Conn, config *ServerConfig) (S type ConnectOptions struct { // UserAgent is the application user agent. UserAgent string + // Authority is the :authority pseudo-header to use. This field has no effect if + // TransportCredentials is set. + Authority string // Dialer specifies how to dial a network address. Dialer func(context.Context, string) (net.Conn, error) // FailOnNonTempDialError specifies if gRPC fails on non-temporary dial errors. @@ -380,18 +437,33 @@ type ConnectOptions struct { PerRPCCredentials []credentials.PerRPCCredentials // TransportCredentials stores the Authenticator required to setup a client connection. TransportCredentials credentials.TransportCredentials + // KeepaliveParams stores the keepalive parameters. + KeepaliveParams keepalive.ClientParameters + // StatsHandler stores the handler for stats. + StatsHandler stats.Handler + // InitialWindowSize sets the initial window size for a stream. + InitialWindowSize int32 + // InitialConnWindowSize sets the initial window size for a connection. + InitialConnWindowSize int32 + // WriteBufferSize sets the size of write buffer which in turn determines how much data can be batched before it's written on the wire. + WriteBufferSize int + // ReadBufferSize sets the size of read buffer, which in turn determines how much data can be read at most for one read syscall. + ReadBufferSize int + // ChannelzParentID sets the addrConn id which initiate the creation of this client transport. + ChannelzParentID int64 } // TargetInfo contains the information of the target such as network address and metadata. type TargetInfo struct { - Addr string - Metadata interface{} + Addr string + Metadata interface{} + Authority string } // NewClientTransport establishes the transport with the required ConnectOptions // and returns it to the caller. -func NewClientTransport(ctx context.Context, target TargetInfo, opts ConnectOptions) (ClientTransport, error) { - return newHTTP2Client(ctx, target, opts) +func NewClientTransport(connectCtx, ctx context.Context, target TargetInfo, opts ConnectOptions, onSuccess func()) (ClientTransport, error) { + return newHTTP2Client(connectCtx, ctx, target, opts, onSuccess) } // Options provides additional hints and information for message @@ -403,7 +475,7 @@ type Options struct { // Delay is a hint to the transport implementation for whether // the data could be buffered for a batching write. The - // Transport implementation may ignore the hint. + // transport implementation may ignore the hint. Delay bool } @@ -415,19 +487,28 @@ type CallHdr struct { // Method specifies the operation to perform. Method string - // RecvCompress specifies the compression algorithm applied on - // inbound messages. - RecvCompress string - // SendCompress specifies the compression algorithm applied on // outbound message. SendCompress string + // Creds specifies credentials.PerRPCCredentials for a call. + Creds credentials.PerRPCCredentials + // Flush indicates whether a new stream command should be sent // to the peer without waiting for the first data. This is - // only a hint. The transport may modify the flush decision + // only a hint. + // If it's true, the transport may modify the flush decision // for performance purposes. + // If it's false, new stream will never be flushed. Flush bool + + // ContentSubtype specifies the content-subtype for a request. For example, a + // content-subtype of "proto" will result in a content-type of + // "application/grpc+proto". The value of ContentSubtype must be all + // lowercase, otherwise the behavior is undefined. See + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests + // for more details. + ContentSubtype string } // ClientTransport is the common interface for all gRPC client-side transport @@ -444,7 +525,7 @@ type ClientTransport interface { // Write sends the data for the given stream. A nil stream indicates // the write is to be performed on the transport as a whole. - Write(s *Stream, data []byte, opts *Options) error + Write(s *Stream, hdr []byte, data []byte, opts *Options) error // NewStream creates a Stream for an RPC. NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, error) @@ -462,10 +543,19 @@ type ClientTransport interface { // once the transport is initiated. Error() <-chan struct{} - // GoAway returns a channel that is closed when ClientTranspor + // GoAway returns a channel that is closed when ClientTransport // receives the draining signal from the server (e.g., GOAWAY frame in // HTTP/2). GoAway() <-chan struct{} + + // GetGoAwayReason returns the reason why GoAway frame was received. + GetGoAwayReason() GoAwayReason + + // IncrMsgSent increments the number of message sent through this transport. + IncrMsgSent() + + // IncrMsgRecv increments the number of message received through this transport. + IncrMsgRecv() } // ServerTransport is the common interface for all gRPC server-side transport @@ -483,12 +573,11 @@ type ServerTransport interface { // Write sends the data for the given stream. // Write may not be called on all streams. - Write(s *Stream, data []byte, opts *Options) error + Write(s *Stream, hdr []byte, data []byte, opts *Options) error - // WriteStatus sends the status of a stream to the client. - // WriteStatus is the final call made on a stream and always - // occurs. - WriteStatus(s *Stream, statusCode codes.Code, statusDesc string) error + // WriteStatus sends the status of a stream to the client. WriteStatus is + // the final call made on a stream and always occurs. + WriteStatus(s *Stream, st *status.Status) error // Close tears down the transport. Once it is called, the transport // should not be accessed any more. All the pending streams and their @@ -500,6 +589,12 @@ type ServerTransport interface { // Drain notifies the client this ServerTransport stops accepting new RPCs. Drain() + + // IncrMsgSent increments the number of message sent through this transport. + IncrMsgSent() + + // IncrMsgRecv increments the number of message received through this transport. + IncrMsgRecv() } // streamErrorf creates an StreamError with the specified error code and description. @@ -549,11 +644,20 @@ func (e ConnectionError) Origin() error { var ( // ErrConnClosing indicates that the transport is closing. ErrConnClosing = connectionErrorf(true, nil, "transport is closing") - // ErrStreamDrain indicates that the stream is rejected by the server because - // the server stops accepting new RPCs. - ErrStreamDrain = streamErrorf(codes.Unavailable, "the server stops accepting new RPCs") + // errStreamDrain indicates that the stream is rejected because the + // connection is draining. This could be caused by goaway or balancer + // removing the address. + errStreamDrain = streamErrorf(codes.Unavailable, "the connection is draining") + // errStreamDone is returned from write at the client side to indiacte application + // layer of an error. + errStreamDone = errors.New("the stream is done") + // StatusGoAway indicates that the server sent a GOAWAY that included this + // stream's ID in unprocessed RPCs. + statusGoAway = status.New(codes.Unavailable, "the stream is rejected because server is draining the connection") ) +// TODO: See if we can replace StreamError with status package errors. + // StreamError is an error that only affects one stream within a connection. type StreamError struct { Code codes.Code @@ -561,44 +665,19 @@ type StreamError struct { } func (e StreamError) Error() string { - return fmt.Sprintf("stream error: code = %d desc = %q", e.Code, e.Desc) + return fmt.Sprintf("stream error: code = %s desc = %q", e.Code, e.Desc) } -// ContextErr converts the error from context package into a StreamError. -func ContextErr(err error) StreamError { - switch err { - case context.DeadlineExceeded: - return streamErrorf(codes.DeadlineExceeded, "%v", err) - case context.Canceled: - return streamErrorf(codes.Canceled, "%v", err) - } - panic(fmt.Sprintf("Unexpected error from context packet: %v", err)) -} +// GoAwayReason contains the reason for the GoAway frame received. +type GoAwayReason uint8 -// wait blocks until it can receive from ctx.Done, closing, or proceed. -// If it receives from ctx.Done, it returns 0, the StreamError for ctx.Err. -// If it receives from done, it returns 0, io.EOF if ctx is not done; otherwise -// it return the StreamError for ctx.Err. -// If it receives from goAway, it returns 0, ErrStreamDrain. -// If it receives from closing, it returns 0, ErrConnClosing. -// If it receives from proceed, it returns the received integer, nil. -func wait(ctx context.Context, done, goAway, closing <-chan struct{}, proceed <-chan int) (int, error) { - select { - case <-ctx.Done(): - return 0, ContextErr(ctx.Err()) - case <-done: - // User cancellation has precedence. - select { - case <-ctx.Done(): - return 0, ContextErr(ctx.Err()) - default: - } - return 0, io.EOF - case <-goAway: - return 0, ErrStreamDrain - case <-closing: - return 0, ErrConnClosing - case i := <-proceed: - return i, nil - } -} +const ( + // GoAwayInvalid indicates that no GoAway frame is received. + GoAwayInvalid GoAwayReason = 0 + // GoAwayNoReason is the default value when GoAway frame is received. + GoAwayNoReason GoAwayReason = 1 + // GoAwayTooManyPings indicates that a GoAway frame with + // ErrCodeEnhanceYourCalm was received and that the debug data said + // "too_many_pings". + GoAwayTooManyPings GoAwayReason = 2 +) diff --git a/vendor/google.golang.org/grpc/vet.sh b/vendor/google.golang.org/grpc/vet.sh new file mode 100755 index 00000000000..6126ab64bd1 --- /dev/null +++ b/vendor/google.golang.org/grpc/vet.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +if [[ `uname -a` = *"Darwin"* ]]; then + echo "It seems you are running on Mac. This script does not work on Mac. See https://github.com/grpc/grpc-go/issues/2047" + exit 1 +fi + +set -ex # Exit on error; debugging enabled. +set -o pipefail # Fail a pipe if any sub-command fails. + +die() { + echo "$@" >&2 + exit 1 +} + +PATH="$GOPATH/bin:$GOROOT/bin:$PATH" + +# Check proto in manual runs or cron runs. +if [[ "$TRAVIS" != "true" || "$TRAVIS_EVENT_TYPE" = "cron" ]]; then + check_proto="true" +fi + +if [ "$1" = "-install" ]; then + go get -d \ + google.golang.org/grpc/... + go get -u \ + github.com/golang/lint/golint \ + golang.org/x/tools/cmd/goimports \ + honnef.co/go/tools/cmd/staticcheck \ + github.com/client9/misspell/cmd/misspell \ + github.com/golang/protobuf/protoc-gen-go + if [[ "$check_proto" = "true" ]]; then + if [[ "$TRAVIS" = "true" ]]; then + PROTOBUF_VERSION=3.3.0 + PROTOC_FILENAME=protoc-${PROTOBUF_VERSION}-linux-x86_64.zip + pushd /home/travis + wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${PROTOC_FILENAME} + unzip ${PROTOC_FILENAME} + bin/protoc --version + popd + elif ! which protoc > /dev/null; then + die "Please install protoc into your path" + fi + fi + exit 0 +elif [[ "$#" -ne 0 ]]; then + die "Unknown argument(s): $*" +fi + +# TODO: Remove this check and the mangling below once "context" is imported +# directly. +if git status --porcelain | read; then + die "Uncommitted or untracked files found; commit changes first" +fi + +git ls-files "*.go" | xargs grep -L "\(Copyright [0-9]\{4,\} gRPC authors\)\|DO NOT EDIT" 2>&1 | tee /dev/stderr | (! read) +git ls-files "*.go" | xargs grep -l "\"unsafe\"" 2>&1 | (! grep -v '_test.go') | tee /dev/stderr | (! read) +gofmt -s -d -l . 2>&1 | tee /dev/stderr | (! read) +goimports -l . 2>&1 | tee /dev/stderr | (! read) +golint ./... 2>&1 | (grep -vE "(_mock|\.pb)\.go:" || true) | tee /dev/stderr | (! read) + +# Undo any edits made by this script. +cleanup() { + git reset --hard HEAD +} +trap cleanup EXIT + +# Rewrite golang.org/x/net/context -> context imports (see grpc/grpc-go#1484). +# TODO: Remove this mangling once "context" is imported directly (grpc/grpc-go#711). +git ls-files "*.go" | xargs sed -i 's:"golang.org/x/net/context":"context":' +set +o pipefail +# TODO: Stop filtering pb.go files once golang/protobuf#214 is fixed. +go tool vet -all . 2>&1 | grep -vE '(clientconn|transport\/transport_test).go:.*cancel (function|var)' | grep -vF '.pb.go:' | tee /dev/stderr | (! read) +set -o pipefail +git reset --hard HEAD + +if [[ "$check_proto" = "true" ]]; then + PATH="/home/travis/bin:$PATH" make proto && \ + git status --porcelain 2>&1 | (! read) || \ + (git status; git --no-pager diff; exit 1) +fi + +# TODO(menghanl): fix errors in transport_test. +staticcheck -ignore ' +google.golang.org/grpc/transport/transport_test.go:SA2002 +google.golang.org/grpc/benchmark/benchmain/main.go:SA1019 +google.golang.org/grpc/stats/stats_test.go:SA1019 +google.golang.org/grpc/test/end2end_test.go:SA1019 +google.golang.org/grpc/balancer_test.go:SA1019 +google.golang.org/grpc/balancer.go:SA1019 +google.golang.org/grpc/clientconn_test.go:SA1019 +' ./... +misspell -error . diff --git a/vendor/vendor.json b/vendor/vendor.json index 6b4ea02c8d7..495806d8d07 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -209,10 +209,10 @@ "revisionTime": "2016-07-11T12:47:28Z" }, { - "checksumSHA1": "kBeNcaKk56FguvPSUCEaH6AxpRc=", + "checksumSHA1": "Pyou8mceOASSFxc7GeXZuVdSMi0=", "path": "github.com/golang/protobuf/proto", - "revision": "8ee79997227bf9b34611aee7946ae64735e6fd93", - "revisionTime": "2016-11-17T03:31:26Z" + "revision": "b4deda0973fb4c70b50d226b1af49f3da59f5265", + "revisionTime": "2018-04-30T18:52:41Z" }, { "checksumSHA1": "g/V4qrXjUGG9B+e3hB+4NAYJ5Gs=", @@ -568,10 +568,10 @@ "revisionTime": "2017-12-21T10:24:07Z" }, { - "checksumSHA1": "Y+HGqEkYM15ir+J93MEaHdyFy0c=", + "checksumSHA1": "GtamqiJoL7PGHsN454AoffBFMa8=", "path": "golang.org/x/net/context", - "revision": "236b8f043b920452504e263bc21d354427127473", - "revisionTime": "2017-02-06T03:21:01Z" + "revision": "640f4622ab692b87c2f3a94265e6f579fe38263d", + "revisionTime": "2018-05-02T16:14:02Z" }, { "checksumSHA1": "vqc3a+oTUGX8PmD0TS+qQ7gmN8I=", @@ -592,16 +592,16 @@ "revisionTime": "2017-02-06T03:21:01Z" }, { - "checksumSHA1": "LC+mzxnIrUS9Kr4s7shpY+A9J2o=", + "checksumSHA1": "j4JQxmxQsZTYj/fbW/7q7g9W50s=", "path": "golang.org/x/net/http2", - "revision": "236b8f043b920452504e263bc21d354427127473", - "revisionTime": "2017-02-06T03:21:01Z" + "revision": "640f4622ab692b87c2f3a94265e6f579fe38263d", + "revisionTime": "2018-05-02T16:14:02Z" }, { - "checksumSHA1": "G3e2/HrQjRy2Ml9lfX0e3AGRWWE=", + "checksumSHA1": "LW///cttbVyQo4Qh11kdIt0VMjs=", "path": "golang.org/x/net/http2/hpack", - "revision": "236b8f043b920452504e263bc21d354427127473", - "revisionTime": "2017-02-06T03:21:01Z" + "revision": "640f4622ab692b87c2f3a94265e6f579fe38263d", + "revisionTime": "2018-05-02T16:14:02Z" }, { "checksumSHA1": "GIGmSrYACByf5JDIP9ByBZksY80=", @@ -650,10 +650,10 @@ "revisionTime": "2017-02-06T03:21:01Z" }, { - "checksumSHA1": "GQHKESPeCcAsnerZPtHadvKUIzs=", + "checksumSHA1": "rJn3m/27kO+2IU6KCCZ74Miby+8=", "path": "golang.org/x/net/trace", - "revision": "236b8f043b920452504e263bc21d354427127473", - "revisionTime": "2017-02-06T03:21:01Z" + "revision": "640f4622ab692b87c2f3a94265e6f579fe38263d", + "revisionTime": "2018-05-02T16:14:02Z" }, { "checksumSHA1": "Xhsm+TevJogC8U4sG6FO+czBMps=", @@ -788,34 +788,58 @@ "revisionTime": "2017-03-01T11:38:50Z" }, { - "checksumSHA1": "+qB6gzApLVQj5Am4zO/a5VP+dCc=", + "checksumSHA1": "BSqEul8NC7Q0aeIkrndbP0ruROw=", "path": "google.golang.org/grpc", - "revision": "708a7f9f3283aa2d4f6132d287d78683babe55c8", - "revisionTime": "2016-12-02T18:43:27Z", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z", "version": "v1.0.5", "versionExact": "v1.0.5" }, { - "checksumSHA1": "08icuA15HRkdYCt6H+Cs90RPQsY=", + "checksumSHA1": "XgBf9DGAD5gsvWR3LLqn3cI5uGQ=", + "path": "google.golang.org/grpc/balancer", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z" + }, + { + "checksumSHA1": "DmKj+1hxJ/0T+e2kwF1EgUaVcQg=", + "path": "google.golang.org/grpc/channelz", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z" + }, + { + "checksumSHA1": "j8Qs+yfgwYYOtodB/1bSlbzV5rs=", "path": "google.golang.org/grpc/codes", - "revision": "708a7f9f3283aa2d4f6132d287d78683babe55c8", - "revisionTime": "2016-12-02T18:43:27Z", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z", "version": "v1.0.5", "versionExact": "v1.0.5" }, { - "checksumSHA1": "Vd1MU+Ojs7GeS6jE52vlxtXvIrI=", + "checksumSHA1": "KthiDKNPHMeIu967enqtE4NaZzI=", "path": "google.golang.org/grpc/credentials", - "revision": "708a7f9f3283aa2d4f6132d287d78683babe55c8", - "revisionTime": "2016-12-02T18:43:27Z", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z", "version": "v1.0.5", "versionExact": "v1.0.5" }, { - "checksumSHA1": "3Lt5hNAG8qJAYSsNghR5uA1zQns=", + "checksumSHA1": "cfLb+pzWB+Glwp82rgfcEST1mv8=", + "path": "google.golang.org/grpc/encoding", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z" + }, + { + "checksumSHA1": "LKKkn7EYA+Do9Qwb2/SUKLFNxoo=", + "path": "google.golang.org/grpc/encoding/proto", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z" + }, + { + "checksumSHA1": "ZPPSFisPDz2ANO4FBZIft+fRxyk=", "path": "google.golang.org/grpc/grpclog", - "revision": "708a7f9f3283aa2d4f6132d287d78683babe55c8", - "revisionTime": "2016-12-02T18:43:27Z", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z", "version": "v1.0.5", "versionExact": "v1.0.5" }, @@ -828,10 +852,10 @@ "versionExact": "v1.0.5" }, { - "checksumSHA1": "XXpD8+S3gLrfmCLOf+RbxblOQkU=", + "checksumSHA1": "OjIAi5AzqlQ7kLtdAyjvdgMf6hc=", "path": "google.golang.org/grpc/metadata", - "revision": "708a7f9f3283aa2d4f6132d287d78683babe55c8", - "revisionTime": "2016-12-02T18:43:27Z", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z", "version": "v1.0.5", "versionExact": "v1.0.5" }, @@ -844,21 +868,27 @@ "versionExact": "v1.0.5" }, { - "checksumSHA1": "3RRoLeH6X2//7tVClOVzxW2bY+E=", + "checksumSHA1": "n5EgDdBqFMa2KQFhtl+FF/4gIFo=", "path": "google.golang.org/grpc/peer", - "revision": "708a7f9f3283aa2d4f6132d287d78683babe55c8", - "revisionTime": "2016-12-02T18:43:27Z", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z", "version": "v1.0.5", "versionExact": "v1.0.5" }, { - "checksumSHA1": "msnYnmpDwizDTD+XLZULYh+1xZw=", + "checksumSHA1": "YclPgme2gT3S0hTkHVdE1zAxJdo=", "path": "google.golang.org/grpc/stats", - "revision": "708a7f9f3283aa2d4f6132d287d78683babe55c8", - "revisionTime": "2016-12-02T18:43:27Z", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z", "version": "v1.0.5", "versionExact": "v1.0.5" }, + { + "checksumSHA1": "FXiovlBmrYdS4QT0Z4nV+x+v5HI=", + "path": "google.golang.org/grpc/status", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z" + }, { "checksumSHA1": "N0TftT6/CyWqp6VRi2DqDx60+Fo=", "path": "google.golang.org/grpc/tap", @@ -868,10 +898,10 @@ "versionExact": "v1.0.5" }, { - "checksumSHA1": "bTCOkTujAY1xcHATTr7q/Et+NWg=", + "checksumSHA1": "Lo7dGbb2DxDJmMIYAgxoBTO5j34=", "path": "google.golang.org/grpc/transport", - "revision": "708a7f9f3283aa2d4f6132d287d78683babe55c8", - "revisionTime": "2016-12-02T18:43:27Z", + "revision": "18957c5fcde0c3037144153d3db03756542007e5", + "revisionTime": "2018-05-04T17:55:21Z", "version": "v1.0.5", "versionExact": "v1.0.5" }, From 2d4d7b85d744b56d60eeb60d35788fbf32f45735 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Fri, 4 May 2018 13:51:46 -0400 Subject: [PATCH 12/54] Access the URL rewriter meta data structure using a pointer This is related to #1662 as it would allow Tyk to keep the compiled regular expression in memory --- api_definition.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api_definition.go b/api_definition.go index 388045df6ec..01d28c0f89a 100644 --- a/api_definition.go +++ b/api_definition.go @@ -105,7 +105,7 @@ type URLSpec struct { InjectHeadersResponse apidef.HeaderInjectionMeta HardTimeout apidef.HardTimeoutMeta CircuitBreaker ExtendedCircuitBreakerMeta - URLRewrite apidef.URLRewriteMeta + URLRewrite *apidef.URLRewriteMeta VirtualPathSpec apidef.VirtualMeta RequestSize apidef.RequestSizeMeta MethodTransform apidef.MethodTransformMeta @@ -678,7 +678,7 @@ func (a APIDefinitionLoader) compileURLRewritesPathSpec(paths []apidef.URLRewrit newSpec := URLSpec{} a.generateRegex(stringSpec.Path, &newSpec, stat) // Extend with method actions - newSpec.URLRewrite = stringSpec + newSpec.URLRewrite = &stringSpec urlSpec = append(urlSpec, newSpec) } @@ -990,7 +990,7 @@ func (a *APISpec) CheckSpecMatchesStatus(r *http.Request, rxPaths []URLSpec, mod } case URLRewrite: if r.Method == v.URLRewrite.Method { - return true, &v.URLRewrite + return true, v.URLRewrite } case VirtualPath: if r.Method == v.VirtualPathSpec.Method { From 3efddec4b71ddddf682d18452973ca8c0ddb94f8 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Fri, 11 May 2018 14:37:02 -0400 Subject: [PATCH 13/54] Update the URL rewriter benchmark to reuse configuration (more realistic) --- mw_url_rewrite_test.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/mw_url_rewrite_test.go b/mw_url_rewrite_test.go index 17358975f89..626ee8b6944 100644 --- a/mw_url_rewrite_test.go +++ b/mw_url_rewrite_test.go @@ -76,16 +76,14 @@ func TestRewriter(t *testing.T) { }) } } - func BenchmarkRewriter(b *testing.B) { b.ReportAllocs() - - for i := 0; i < b.N; i++ { - for _, tc := range testRewriterData { - testConf := apidef.URLRewriteMeta{ - MatchPattern: tc.pattern, - RewriteTo: tc.to, - } + for _, tc := range testRewriterData { + testConf := apidef.URLRewriteMeta{ + MatchPattern: tc.pattern, + RewriteTo: tc.to, + } + for i := 0; i < b.N; i++ { r := httptest.NewRequest("GET", tc.in, nil) urlRewrite(&testConf, r) } From e353a703fb2c0506998cc68f31ee0f22d64b4036 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 22 May 2018 14:49:10 +0300 Subject: [PATCH 14/54] Fix GRPC vendoring --- vendor/github.com/golang/protobuf/LICENSE | 3 - .../github.com/golang/protobuf/ptypes/any.go | 141 + .../golang/protobuf/ptypes/any/any.pb.go | 191 + .../golang/protobuf/ptypes/any/any.proto | 149 + .../github.com/golang/protobuf/ptypes/doc.go | 35 + .../golang/protobuf/ptypes/duration.go | 102 + .../protobuf/ptypes/duration/duration.pb.go | 159 + .../protobuf/ptypes/duration/duration.proto | 117 + .../golang/protobuf/ptypes/timestamp.go | 134 + .../protobuf/ptypes/timestamp/timestamp.pb.go | 175 + .../protobuf/ptypes/timestamp/timestamp.proto | 133 + vendor/golang.org/x/net/http/httpguts/guts.go | 65 + .../golang.org/x/net/http/httpguts/httplex.go | 346 + .../x/text/secure/bidirule/bidirule.go | 342 + vendor/golang.org/x/text/unicode/bidi/bidi.go | 198 + .../golang.org/x/text/unicode/bidi/bracket.go | 335 + vendor/golang.org/x/text/unicode/bidi/core.go | 1058 +++ vendor/golang.org/x/text/unicode/bidi/gen.go | 133 + .../x/text/unicode/bidi/gen_ranges.go | 57 + .../x/text/unicode/bidi/gen_trieval.go | 64 + vendor/golang.org/x/text/unicode/bidi/prop.go | 206 + .../golang.org/x/text/unicode/bidi/tables.go | 1779 ++++ .../golang.org/x/text/unicode/bidi/trieval.go | 60 + .../x/text/unicode/norm/composition.go | 514 ++ .../x/text/unicode/norm/forminfo.go | 256 + .../golang.org/x/text/unicode/norm/input.go | 105 + vendor/golang.org/x/text/unicode/norm/iter.go | 450 + .../x/text/unicode/norm/maketables.go | 978 +++ .../x/text/unicode/norm/normalize.go | 608 ++ .../x/text/unicode/norm/readwriter.go | 125 + .../golang.org/x/text/unicode/norm/tables.go | 7627 +++++++++++++++++ .../x/text/unicode/norm/transform.go | 88 + vendor/golang.org/x/text/unicode/norm/trie.go | 54 + .../golang.org/x/text/unicode/norm/triegen.go | 117 + vendor/google.golang.org/genproto/LICENSE | 202 + .../googleapis/rpc/status/status.pb.go | 156 + .../grpc/balancer/base/balancer.go | 208 + .../grpc/balancer/base/base.go | 52 + .../grpc/balancer/roundrobin/roundrobin.go | 79 + .../grpc/connectivity/connectivity.go | 72 + .../grpclb/grpc_lb_v1/messages/messages.pb.go | 799 ++ .../grpclb/grpc_lb_v1/messages/messages.proto | 155 + .../grpc/keepalive/keepalive.go | 65 + .../grpc/resolver/dns/dns_resolver.go | 379 + .../grpc/resolver/dns/go17.go | 35 + .../grpc/resolver/dns/go18.go | 29 + .../grpc/resolver/passthrough/passthrough.go | 57 + .../grpc/resolver/resolver.go | 154 + vendor/vendor.json | 102 + 49 files changed, 19445 insertions(+), 3 deletions(-) create mode 100644 vendor/github.com/golang/protobuf/ptypes/any.go create mode 100644 vendor/github.com/golang/protobuf/ptypes/any/any.pb.go create mode 100644 vendor/github.com/golang/protobuf/ptypes/any/any.proto create mode 100644 vendor/github.com/golang/protobuf/ptypes/doc.go create mode 100644 vendor/github.com/golang/protobuf/ptypes/duration.go create mode 100644 vendor/github.com/golang/protobuf/ptypes/duration/duration.pb.go create mode 100644 vendor/github.com/golang/protobuf/ptypes/duration/duration.proto create mode 100644 vendor/github.com/golang/protobuf/ptypes/timestamp.go create mode 100644 vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.pb.go create mode 100644 vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.proto create mode 100644 vendor/golang.org/x/net/http/httpguts/guts.go create mode 100644 vendor/golang.org/x/net/http/httpguts/httplex.go create mode 100644 vendor/golang.org/x/text/secure/bidirule/bidirule.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/bidi.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/bracket.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/core.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/gen.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/gen_ranges.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/gen_trieval.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/prop.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/tables.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/trieval.go create mode 100644 vendor/golang.org/x/text/unicode/norm/composition.go create mode 100644 vendor/golang.org/x/text/unicode/norm/forminfo.go create mode 100644 vendor/golang.org/x/text/unicode/norm/input.go create mode 100644 vendor/golang.org/x/text/unicode/norm/iter.go create mode 100644 vendor/golang.org/x/text/unicode/norm/maketables.go create mode 100644 vendor/golang.org/x/text/unicode/norm/normalize.go create mode 100644 vendor/golang.org/x/text/unicode/norm/readwriter.go create mode 100644 vendor/golang.org/x/text/unicode/norm/tables.go create mode 100644 vendor/golang.org/x/text/unicode/norm/transform.go create mode 100644 vendor/golang.org/x/text/unicode/norm/trie.go create mode 100644 vendor/golang.org/x/text/unicode/norm/triegen.go create mode 100644 vendor/google.golang.org/genproto/LICENSE create mode 100644 vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go create mode 100644 vendor/google.golang.org/grpc/balancer/base/balancer.go create mode 100644 vendor/google.golang.org/grpc/balancer/base/base.go create mode 100644 vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go create mode 100644 vendor/google.golang.org/grpc/connectivity/connectivity.go create mode 100644 vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.pb.go create mode 100644 vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.proto create mode 100644 vendor/google.golang.org/grpc/keepalive/keepalive.go create mode 100644 vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go create mode 100644 vendor/google.golang.org/grpc/resolver/dns/go17.go create mode 100644 vendor/google.golang.org/grpc/resolver/dns/go18.go create mode 100644 vendor/google.golang.org/grpc/resolver/passthrough/passthrough.go create mode 100644 vendor/google.golang.org/grpc/resolver/resolver.go diff --git a/vendor/github.com/golang/protobuf/LICENSE b/vendor/github.com/golang/protobuf/LICENSE index 1b1b1921efa..0f646931a46 100644 --- a/vendor/github.com/golang/protobuf/LICENSE +++ b/vendor/github.com/golang/protobuf/LICENSE @@ -1,7 +1,4 @@ -Go support for Protocol Buffers - Google's data interchange format - Copyright 2010 The Go Authors. All rights reserved. -https://github.com/golang/protobuf Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/vendor/github.com/golang/protobuf/ptypes/any.go b/vendor/github.com/golang/protobuf/ptypes/any.go new file mode 100644 index 00000000000..70276e8f5c9 --- /dev/null +++ b/vendor/github.com/golang/protobuf/ptypes/any.go @@ -0,0 +1,141 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package ptypes + +// This file implements functions to marshal proto.Message to/from +// google.protobuf.Any message. + +import ( + "fmt" + "reflect" + "strings" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes/any" +) + +const googleApis = "type.googleapis.com/" + +// AnyMessageName returns the name of the message contained in a google.protobuf.Any message. +// +// Note that regular type assertions should be done using the Is +// function. AnyMessageName is provided for less common use cases like filtering a +// sequence of Any messages based on a set of allowed message type names. +func AnyMessageName(any *any.Any) (string, error) { + if any == nil { + return "", fmt.Errorf("message is nil") + } + slash := strings.LastIndex(any.TypeUrl, "/") + if slash < 0 { + return "", fmt.Errorf("message type url %q is invalid", any.TypeUrl) + } + return any.TypeUrl[slash+1:], nil +} + +// MarshalAny takes the protocol buffer and encodes it into google.protobuf.Any. +func MarshalAny(pb proto.Message) (*any.Any, error) { + value, err := proto.Marshal(pb) + if err != nil { + return nil, err + } + return &any.Any{TypeUrl: googleApis + proto.MessageName(pb), Value: value}, nil +} + +// DynamicAny is a value that can be passed to UnmarshalAny to automatically +// allocate a proto.Message for the type specified in a google.protobuf.Any +// message. The allocated message is stored in the embedded proto.Message. +// +// Example: +// +// var x ptypes.DynamicAny +// if err := ptypes.UnmarshalAny(a, &x); err != nil { ... } +// fmt.Printf("unmarshaled message: %v", x.Message) +type DynamicAny struct { + proto.Message +} + +// Empty returns a new proto.Message of the type specified in a +// google.protobuf.Any message. It returns an error if corresponding message +// type isn't linked in. +func Empty(any *any.Any) (proto.Message, error) { + aname, err := AnyMessageName(any) + if err != nil { + return nil, err + } + + t := proto.MessageType(aname) + if t == nil { + return nil, fmt.Errorf("any: message type %q isn't linked in", aname) + } + return reflect.New(t.Elem()).Interface().(proto.Message), nil +} + +// UnmarshalAny parses the protocol buffer representation in a google.protobuf.Any +// message and places the decoded result in pb. It returns an error if type of +// contents of Any message does not match type of pb message. +// +// pb can be a proto.Message, or a *DynamicAny. +func UnmarshalAny(any *any.Any, pb proto.Message) error { + if d, ok := pb.(*DynamicAny); ok { + if d.Message == nil { + var err error + d.Message, err = Empty(any) + if err != nil { + return err + } + } + return UnmarshalAny(any, d.Message) + } + + aname, err := AnyMessageName(any) + if err != nil { + return err + } + + mname := proto.MessageName(pb) + if aname != mname { + return fmt.Errorf("mismatched message type: got %q want %q", aname, mname) + } + return proto.Unmarshal(any.Value, pb) +} + +// Is returns true if any value contains a given message type. +func Is(any *any.Any, pb proto.Message) bool { + // The following is equivalent to AnyMessageName(any) == proto.MessageName(pb), + // but it avoids scanning TypeUrl for the slash. + if any == nil { + return false + } + name := proto.MessageName(pb) + prefix := len(any.TypeUrl) - len(name) + return prefix >= 1 && any.TypeUrl[prefix-1] == '/' && any.TypeUrl[prefix:] == name +} diff --git a/vendor/github.com/golang/protobuf/ptypes/any/any.pb.go b/vendor/github.com/golang/protobuf/ptypes/any/any.pb.go new file mode 100644 index 00000000000..f67edc7dc2b --- /dev/null +++ b/vendor/github.com/golang/protobuf/ptypes/any/any.pb.go @@ -0,0 +1,191 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: google/protobuf/any.proto + +package any // import "github.com/golang/protobuf/ptypes/any" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// Example 4: Pack and unpack a message in Go +// +// foo := &pb.Foo{...} +// any, err := ptypes.MarshalAny(foo) +// ... +// foo := &pb.Foo{} +// if err := ptypes.UnmarshalAny(any, foo); err != nil { +// ... +// } +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// +// JSON +// ==== +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +type Any struct { + // A URL/resource name whose content describes the type of the + // serialized protocol buffer message. + // + // For URLs which use the scheme `http`, `https`, or no scheme, the + // following restrictions and interpretations apply: + // + // * If no scheme is provided, `https` is assumed. + // * The last segment of the URL's path must represent the fully + // qualified name of the type (as in `path/google.protobuf.Duration`). + // The name should be in a canonical form (e.g., leading "." is + // not accepted). + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + // + TypeUrl string `protobuf:"bytes,1,opt,name=type_url,json=typeUrl" json:"type_url,omitempty"` + // Must be a valid serialized protocol buffer of the above specified type. + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Any) Reset() { *m = Any{} } +func (m *Any) String() string { return proto.CompactTextString(m) } +func (*Any) ProtoMessage() {} +func (*Any) Descriptor() ([]byte, []int) { + return fileDescriptor_any_744b9ca530f228db, []int{0} +} +func (*Any) XXX_WellKnownType() string { return "Any" } +func (m *Any) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Any.Unmarshal(m, b) +} +func (m *Any) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Any.Marshal(b, m, deterministic) +} +func (dst *Any) XXX_Merge(src proto.Message) { + xxx_messageInfo_Any.Merge(dst, src) +} +func (m *Any) XXX_Size() int { + return xxx_messageInfo_Any.Size(m) +} +func (m *Any) XXX_DiscardUnknown() { + xxx_messageInfo_Any.DiscardUnknown(m) +} + +var xxx_messageInfo_Any proto.InternalMessageInfo + +func (m *Any) GetTypeUrl() string { + if m != nil { + return m.TypeUrl + } + return "" +} + +func (m *Any) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +func init() { + proto.RegisterType((*Any)(nil), "google.protobuf.Any") +} + +func init() { proto.RegisterFile("google/protobuf/any.proto", fileDescriptor_any_744b9ca530f228db) } + +var fileDescriptor_any_744b9ca530f228db = []byte{ + // 185 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4c, 0xcf, 0xcf, 0x4f, + 0xcf, 0x49, 0xd5, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x4f, 0x2a, 0x4d, 0xd3, 0x4f, 0xcc, 0xab, 0xd4, + 0x03, 0x73, 0x84, 0xf8, 0x21, 0x52, 0x7a, 0x30, 0x29, 0x25, 0x33, 0x2e, 0x66, 0xc7, 0xbc, 0x4a, + 0x21, 0x49, 0x2e, 0x8e, 0x92, 0xca, 0x82, 0xd4, 0xf8, 0xd2, 0xa2, 0x1c, 0x09, 0x46, 0x05, 0x46, + 0x0d, 0xce, 0x20, 0x76, 0x10, 0x3f, 0xb4, 0x28, 0x47, 0x48, 0x84, 0x8b, 0xb5, 0x2c, 0x31, 0xa7, + 0x34, 0x55, 0x82, 0x49, 0x81, 0x51, 0x83, 0x27, 0x08, 0xc2, 0x71, 0xca, 0xe7, 0x12, 0x4e, 0xce, + 0xcf, 0xd5, 0x43, 0x33, 0xce, 0x89, 0xc3, 0x31, 0xaf, 0x32, 0x00, 0xc4, 0x09, 0x60, 0x8c, 0x52, + 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0xcf, 0x49, 0xcc, + 0x4b, 0x47, 0xb8, 0xa8, 0x00, 0x64, 0x7a, 0x31, 0xc8, 0x61, 0x8b, 0x98, 0x98, 0xdd, 0x03, 0x9c, + 0x56, 0x31, 0xc9, 0xb9, 0x43, 0x8c, 0x0a, 0x80, 0x2a, 0xd1, 0x0b, 0x4f, 0xcd, 0xc9, 0xf1, 0xce, + 0xcb, 0x2f, 0xcf, 0x0b, 0x01, 0x29, 0x4d, 0x62, 0x03, 0xeb, 0x35, 0x06, 0x04, 0x00, 0x00, 0xff, + 0xff, 0x13, 0xf8, 0xe8, 0x42, 0xdd, 0x00, 0x00, 0x00, +} diff --git a/vendor/github.com/golang/protobuf/ptypes/any/any.proto b/vendor/github.com/golang/protobuf/ptypes/any/any.proto new file mode 100644 index 00000000000..c7486676231 --- /dev/null +++ b/vendor/github.com/golang/protobuf/ptypes/any/any.proto @@ -0,0 +1,149 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "github.com/golang/protobuf/ptypes/any"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// Example 4: Pack and unpack a message in Go +// +// foo := &pb.Foo{...} +// any, err := ptypes.MarshalAny(foo) +// ... +// foo := &pb.Foo{} +// if err := ptypes.UnmarshalAny(any, foo); err != nil { +// ... +// } +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// +// JSON +// ==== +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name whose content describes the type of the + // serialized protocol buffer message. + // + // For URLs which use the scheme `http`, `https`, or no scheme, the + // following restrictions and interpretations apply: + // + // * If no scheme is provided, `https` is assumed. + // * The last segment of the URL's path must represent the fully + // qualified name of the type (as in `path/google.protobuf.Duration`). + // The name should be in a canonical form (e.g., leading "." is + // not accepted). + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + // + string type_url = 1; + + // Must be a valid serialized protocol buffer of the above specified type. + bytes value = 2; +} diff --git a/vendor/github.com/golang/protobuf/ptypes/doc.go b/vendor/github.com/golang/protobuf/ptypes/doc.go new file mode 100644 index 00000000000..c0d595da7ab --- /dev/null +++ b/vendor/github.com/golang/protobuf/ptypes/doc.go @@ -0,0 +1,35 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* +Package ptypes contains code for interacting with well-known types. +*/ +package ptypes diff --git a/vendor/github.com/golang/protobuf/ptypes/duration.go b/vendor/github.com/golang/protobuf/ptypes/duration.go new file mode 100644 index 00000000000..65cb0f8eb5f --- /dev/null +++ b/vendor/github.com/golang/protobuf/ptypes/duration.go @@ -0,0 +1,102 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package ptypes + +// This file implements conversions between google.protobuf.Duration +// and time.Duration. + +import ( + "errors" + "fmt" + "time" + + durpb "github.com/golang/protobuf/ptypes/duration" +) + +const ( + // Range of a durpb.Duration in seconds, as specified in + // google/protobuf/duration.proto. This is about 10,000 years in seconds. + maxSeconds = int64(10000 * 365.25 * 24 * 60 * 60) + minSeconds = -maxSeconds +) + +// validateDuration determines whether the durpb.Duration is valid according to the +// definition in google/protobuf/duration.proto. A valid durpb.Duration +// may still be too large to fit into a time.Duration (the range of durpb.Duration +// is about 10,000 years, and the range of time.Duration is about 290). +func validateDuration(d *durpb.Duration) error { + if d == nil { + return errors.New("duration: nil Duration") + } + if d.Seconds < minSeconds || d.Seconds > maxSeconds { + return fmt.Errorf("duration: %v: seconds out of range", d) + } + if d.Nanos <= -1e9 || d.Nanos >= 1e9 { + return fmt.Errorf("duration: %v: nanos out of range", d) + } + // Seconds and Nanos must have the same sign, unless d.Nanos is zero. + if (d.Seconds < 0 && d.Nanos > 0) || (d.Seconds > 0 && d.Nanos < 0) { + return fmt.Errorf("duration: %v: seconds and nanos have different signs", d) + } + return nil +} + +// Duration converts a durpb.Duration to a time.Duration. Duration +// returns an error if the durpb.Duration is invalid or is too large to be +// represented in a time.Duration. +func Duration(p *durpb.Duration) (time.Duration, error) { + if err := validateDuration(p); err != nil { + return 0, err + } + d := time.Duration(p.Seconds) * time.Second + if int64(d/time.Second) != p.Seconds { + return 0, fmt.Errorf("duration: %v is out of range for time.Duration", p) + } + if p.Nanos != 0 { + d += time.Duration(p.Nanos) + if (d < 0) != (p.Nanos < 0) { + return 0, fmt.Errorf("duration: %v is out of range for time.Duration", p) + } + } + return d, nil +} + +// DurationProto converts a time.Duration to a durpb.Duration. +func DurationProto(d time.Duration) *durpb.Duration { + nanos := d.Nanoseconds() + secs := nanos / 1e9 + nanos -= secs * 1e9 + return &durpb.Duration{ + Seconds: secs, + Nanos: int32(nanos), + } +} diff --git a/vendor/github.com/golang/protobuf/ptypes/duration/duration.pb.go b/vendor/github.com/golang/protobuf/ptypes/duration/duration.pb.go new file mode 100644 index 00000000000..4d75473b8b2 --- /dev/null +++ b/vendor/github.com/golang/protobuf/ptypes/duration/duration.pb.go @@ -0,0 +1,159 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: google/protobuf/duration.proto + +package duration // import "github.com/golang/protobuf/ptypes/duration" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// # Examples +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (durations.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +// Example 3: Compute Duration from datetime.timedelta in Python. +// +// td = datetime.timedelta(days=3, minutes=10) +// duration = Duration() +// duration.FromTimedelta(td) +// +// # JSON Mapping +// +// In JSON format, the Duration type is encoded as a string rather than an +// object, where the string ends in the suffix "s" (indicating seconds) and +// is preceded by the number of seconds, with nanoseconds expressed as +// fractional seconds. For example, 3 seconds with 0 nanoseconds should be +// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should +// be expressed in JSON format as "3.000000001s", and 3 seconds and 1 +// microsecond should be expressed in JSON format as "3.000001s". +// +// +type Duration struct { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. Note: these bounds are computed from: + // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years + Seconds int64 `protobuf:"varint,1,opt,name=seconds" json:"seconds,omitempty"` + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + Nanos int32 `protobuf:"varint,2,opt,name=nanos" json:"nanos,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Duration) Reset() { *m = Duration{} } +func (m *Duration) String() string { return proto.CompactTextString(m) } +func (*Duration) ProtoMessage() {} +func (*Duration) Descriptor() ([]byte, []int) { + return fileDescriptor_duration_e7d612259e3f0613, []int{0} +} +func (*Duration) XXX_WellKnownType() string { return "Duration" } +func (m *Duration) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Duration.Unmarshal(m, b) +} +func (m *Duration) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Duration.Marshal(b, m, deterministic) +} +func (dst *Duration) XXX_Merge(src proto.Message) { + xxx_messageInfo_Duration.Merge(dst, src) +} +func (m *Duration) XXX_Size() int { + return xxx_messageInfo_Duration.Size(m) +} +func (m *Duration) XXX_DiscardUnknown() { + xxx_messageInfo_Duration.DiscardUnknown(m) +} + +var xxx_messageInfo_Duration proto.InternalMessageInfo + +func (m *Duration) GetSeconds() int64 { + if m != nil { + return m.Seconds + } + return 0 +} + +func (m *Duration) GetNanos() int32 { + if m != nil { + return m.Nanos + } + return 0 +} + +func init() { + proto.RegisterType((*Duration)(nil), "google.protobuf.Duration") +} + +func init() { + proto.RegisterFile("google/protobuf/duration.proto", fileDescriptor_duration_e7d612259e3f0613) +} + +var fileDescriptor_duration_e7d612259e3f0613 = []byte{ + // 190 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4b, 0xcf, 0xcf, 0x4f, + 0xcf, 0x49, 0xd5, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x4f, 0x2a, 0x4d, 0xd3, 0x4f, 0x29, 0x2d, 0x4a, + 0x2c, 0xc9, 0xcc, 0xcf, 0xd3, 0x03, 0x8b, 0x08, 0xf1, 0x43, 0xe4, 0xf5, 0x60, 0xf2, 0x4a, 0x56, + 0x5c, 0x1c, 0x2e, 0x50, 0x25, 0x42, 0x12, 0x5c, 0xec, 0xc5, 0xa9, 0xc9, 0xf9, 0x79, 0x29, 0xc5, + 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0xcc, 0x41, 0x30, 0xae, 0x90, 0x08, 0x17, 0x6b, 0x5e, 0x62, 0x5e, + 0x7e, 0xb1, 0x04, 0x93, 0x02, 0xa3, 0x06, 0x6b, 0x10, 0x84, 0xe3, 0x54, 0xc3, 0x25, 0x9c, 0x9c, + 0x9f, 0xab, 0x87, 0x66, 0xa4, 0x13, 0x2f, 0xcc, 0xc0, 0x00, 0x90, 0x48, 0x00, 0x63, 0x94, 0x56, + 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, 0x5e, 0x72, 0x7e, 0xae, 0x7e, 0x7a, 0x7e, 0x4e, 0x62, 0x5e, + 0x3a, 0xc2, 0x7d, 0x05, 0x25, 0x95, 0x05, 0xa9, 0xc5, 0x70, 0x67, 0xfe, 0x60, 0x64, 0x5c, 0xc4, + 0xc4, 0xec, 0x1e, 0xe0, 0xb4, 0x8a, 0x49, 0xce, 0x1d, 0x62, 0x6e, 0x00, 0x54, 0xa9, 0x5e, 0x78, + 0x6a, 0x4e, 0x8e, 0x77, 0x5e, 0x7e, 0x79, 0x5e, 0x08, 0x48, 0x4b, 0x12, 0x1b, 0xd8, 0x0c, 0x63, + 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xdc, 0x84, 0x30, 0xff, 0xf3, 0x00, 0x00, 0x00, +} diff --git a/vendor/github.com/golang/protobuf/ptypes/duration/duration.proto b/vendor/github.com/golang/protobuf/ptypes/duration/duration.proto new file mode 100644 index 00000000000..975fce41aae --- /dev/null +++ b/vendor/github.com/golang/protobuf/ptypes/duration/duration.proto @@ -0,0 +1,117 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "github.com/golang/protobuf/ptypes/duration"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DurationProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// # Examples +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (durations.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +// Example 3: Compute Duration from datetime.timedelta in Python. +// +// td = datetime.timedelta(days=3, minutes=10) +// duration = Duration() +// duration.FromTimedelta(td) +// +// # JSON Mapping +// +// In JSON format, the Duration type is encoded as a string rather than an +// object, where the string ends in the suffix "s" (indicating seconds) and +// is preceded by the number of seconds, with nanoseconds expressed as +// fractional seconds. For example, 3 seconds with 0 nanoseconds should be +// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should +// be expressed in JSON format as "3.000000001s", and 3 seconds and 1 +// microsecond should be expressed in JSON format as "3.000001s". +// +// +message Duration { + + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. Note: these bounds are computed from: + // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} diff --git a/vendor/github.com/golang/protobuf/ptypes/timestamp.go b/vendor/github.com/golang/protobuf/ptypes/timestamp.go new file mode 100644 index 00000000000..47f10dbc2cc --- /dev/null +++ b/vendor/github.com/golang/protobuf/ptypes/timestamp.go @@ -0,0 +1,134 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package ptypes + +// This file implements operations on google.protobuf.Timestamp. + +import ( + "errors" + "fmt" + "time" + + tspb "github.com/golang/protobuf/ptypes/timestamp" +) + +const ( + // Seconds field of the earliest valid Timestamp. + // This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix(). + minValidSeconds = -62135596800 + // Seconds field just after the latest valid Timestamp. + // This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix(). + maxValidSeconds = 253402300800 +) + +// validateTimestamp determines whether a Timestamp is valid. +// A valid timestamp represents a time in the range +// [0001-01-01, 10000-01-01) and has a Nanos field +// in the range [0, 1e9). +// +// If the Timestamp is valid, validateTimestamp returns nil. +// Otherwise, it returns an error that describes +// the problem. +// +// Every valid Timestamp can be represented by a time.Time, but the converse is not true. +func validateTimestamp(ts *tspb.Timestamp) error { + if ts == nil { + return errors.New("timestamp: nil Timestamp") + } + if ts.Seconds < minValidSeconds { + return fmt.Errorf("timestamp: %v before 0001-01-01", ts) + } + if ts.Seconds >= maxValidSeconds { + return fmt.Errorf("timestamp: %v after 10000-01-01", ts) + } + if ts.Nanos < 0 || ts.Nanos >= 1e9 { + return fmt.Errorf("timestamp: %v: nanos not in range [0, 1e9)", ts) + } + return nil +} + +// Timestamp converts a google.protobuf.Timestamp proto to a time.Time. +// It returns an error if the argument is invalid. +// +// Unlike most Go functions, if Timestamp returns an error, the first return value +// is not the zero time.Time. Instead, it is the value obtained from the +// time.Unix function when passed the contents of the Timestamp, in the UTC +// locale. This may or may not be a meaningful time; many invalid Timestamps +// do map to valid time.Times. +// +// A nil Timestamp returns an error. The first return value in that case is +// undefined. +func Timestamp(ts *tspb.Timestamp) (time.Time, error) { + // Don't return the zero value on error, because corresponds to a valid + // timestamp. Instead return whatever time.Unix gives us. + var t time.Time + if ts == nil { + t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp + } else { + t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC() + } + return t, validateTimestamp(ts) +} + +// TimestampNow returns a google.protobuf.Timestamp for the current time. +func TimestampNow() *tspb.Timestamp { + ts, err := TimestampProto(time.Now()) + if err != nil { + panic("ptypes: time.Now() out of Timestamp range") + } + return ts +} + +// TimestampProto converts the time.Time to a google.protobuf.Timestamp proto. +// It returns an error if the resulting Timestamp is invalid. +func TimestampProto(t time.Time) (*tspb.Timestamp, error) { + seconds := t.Unix() + nanos := int32(t.Sub(time.Unix(seconds, 0))) + ts := &tspb.Timestamp{ + Seconds: seconds, + Nanos: nanos, + } + if err := validateTimestamp(ts); err != nil { + return nil, err + } + return ts, nil +} + +// TimestampString returns the RFC 3339 string for valid Timestamps. For invalid +// Timestamps, it returns an error message in parentheses. +func TimestampString(ts *tspb.Timestamp) string { + t, err := Timestamp(ts) + if err != nil { + return fmt.Sprintf("(%v)", err) + } + return t.Format(time.RFC3339Nano) +} diff --git a/vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.pb.go b/vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.pb.go new file mode 100644 index 00000000000..e9c22228216 --- /dev/null +++ b/vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.pb.go @@ -0,0 +1,175 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: google/protobuf/timestamp.proto + +package timestamp // import "github.com/golang/protobuf/ptypes/timestamp" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// A Timestamp represents a point in time independent of any time zone +// or calendar, represented as seconds and fractions of seconds at +// nanosecond resolution in UTC Epoch time. It is encoded using the +// Proleptic Gregorian Calendar which extends the Gregorian calendar +// backwards to year one. It is encoded assuming all minutes are 60 +// seconds long, i.e. leap seconds are "smeared" so that no leap second +// table is needed for interpretation. Range is from +// 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. +// By restricting to that range, we ensure that we can convert to +// and from RFC 3339 date strings. +// See [https://www.ietf.org/rfc/rfc3339.txt](https://www.ietf.org/rfc/rfc3339.txt). +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required, though only UTC (as indicated by "Z") is presently supported. +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString] +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) +// with the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one +// can use the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime--) +// to obtain a formatter capable of generating timestamps in this format. +// +// +type Timestamp struct { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + Seconds int64 `protobuf:"varint,1,opt,name=seconds" json:"seconds,omitempty"` + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + Nanos int32 `protobuf:"varint,2,opt,name=nanos" json:"nanos,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Timestamp) Reset() { *m = Timestamp{} } +func (m *Timestamp) String() string { return proto.CompactTextString(m) } +func (*Timestamp) ProtoMessage() {} +func (*Timestamp) Descriptor() ([]byte, []int) { + return fileDescriptor_timestamp_b826e8e5fba671a8, []int{0} +} +func (*Timestamp) XXX_WellKnownType() string { return "Timestamp" } +func (m *Timestamp) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Timestamp.Unmarshal(m, b) +} +func (m *Timestamp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Timestamp.Marshal(b, m, deterministic) +} +func (dst *Timestamp) XXX_Merge(src proto.Message) { + xxx_messageInfo_Timestamp.Merge(dst, src) +} +func (m *Timestamp) XXX_Size() int { + return xxx_messageInfo_Timestamp.Size(m) +} +func (m *Timestamp) XXX_DiscardUnknown() { + xxx_messageInfo_Timestamp.DiscardUnknown(m) +} + +var xxx_messageInfo_Timestamp proto.InternalMessageInfo + +func (m *Timestamp) GetSeconds() int64 { + if m != nil { + return m.Seconds + } + return 0 +} + +func (m *Timestamp) GetNanos() int32 { + if m != nil { + return m.Nanos + } + return 0 +} + +func init() { + proto.RegisterType((*Timestamp)(nil), "google.protobuf.Timestamp") +} + +func init() { + proto.RegisterFile("google/protobuf/timestamp.proto", fileDescriptor_timestamp_b826e8e5fba671a8) +} + +var fileDescriptor_timestamp_b826e8e5fba671a8 = []byte{ + // 191 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4f, 0xcf, 0xcf, 0x4f, + 0xcf, 0x49, 0xd5, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x4f, 0x2a, 0x4d, 0xd3, 0x2f, 0xc9, 0xcc, 0x4d, + 0x2d, 0x2e, 0x49, 0xcc, 0x2d, 0xd0, 0x03, 0x0b, 0x09, 0xf1, 0x43, 0x14, 0xe8, 0xc1, 0x14, 0x28, + 0x59, 0x73, 0x71, 0x86, 0xc0, 0xd4, 0x08, 0x49, 0x70, 0xb1, 0x17, 0xa7, 0x26, 0xe7, 0xe7, 0xa5, + 0x14, 0x4b, 0x30, 0x2a, 0x30, 0x6a, 0x30, 0x07, 0xc1, 0xb8, 0x42, 0x22, 0x5c, 0xac, 0x79, 0x89, + 0x79, 0xf9, 0xc5, 0x12, 0x4c, 0x0a, 0x8c, 0x1a, 0xac, 0x41, 0x10, 0x8e, 0x53, 0x1d, 0x97, 0x70, + 0x72, 0x7e, 0xae, 0x1e, 0x9a, 0x99, 0x4e, 0x7c, 0x70, 0x13, 0x03, 0x40, 0x42, 0x01, 0x8c, 0x51, + 0xda, 0xe9, 0x99, 0x25, 0x19, 0xa5, 0x49, 0x7a, 0xc9, 0xf9, 0xb9, 0xfa, 0xe9, 0xf9, 0x39, 0x89, + 0x79, 0xe9, 0x08, 0x27, 0x16, 0x94, 0x54, 0x16, 0xa4, 0x16, 0x23, 0x5c, 0xfa, 0x83, 0x91, 0x71, + 0x11, 0x13, 0xb3, 0x7b, 0x80, 0xd3, 0x2a, 0x26, 0x39, 0x77, 0x88, 0xc9, 0x01, 0x50, 0xb5, 0x7a, + 0xe1, 0xa9, 0x39, 0x39, 0xde, 0x79, 0xf9, 0xe5, 0x79, 0x21, 0x20, 0x3d, 0x49, 0x6c, 0x60, 0x43, + 0x8c, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xbc, 0x77, 0x4a, 0x07, 0xf7, 0x00, 0x00, 0x00, +} diff --git a/vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.proto b/vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.proto new file mode 100644 index 00000000000..06750ab1f12 --- /dev/null +++ b/vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.proto @@ -0,0 +1,133 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "github.com/golang/protobuf/ptypes/timestamp"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone +// or calendar, represented as seconds and fractions of seconds at +// nanosecond resolution in UTC Epoch time. It is encoded using the +// Proleptic Gregorian Calendar which extends the Gregorian calendar +// backwards to year one. It is encoded assuming all minutes are 60 +// seconds long, i.e. leap seconds are "smeared" so that no leap second +// table is needed for interpretation. Range is from +// 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. +// By restricting to that range, we ensure that we can convert to +// and from RFC 3339 date strings. +// See [https://www.ietf.org/rfc/rfc3339.txt](https://www.ietf.org/rfc/rfc3339.txt). +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required, though only UTC (as indicated by "Z") is presently supported. +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString] +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) +// with the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one +// can use the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime--) +// to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/vendor/golang.org/x/net/http/httpguts/guts.go b/vendor/golang.org/x/net/http/httpguts/guts.go new file mode 100644 index 00000000000..8255fd49b42 --- /dev/null +++ b/vendor/golang.org/x/net/http/httpguts/guts.go @@ -0,0 +1,65 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httpguts provides functions implementing various details +// of the HTTP specification. +// +// This package is shared by the standard library (which vendors it) +// and x/net/http2. It comes with no API stability promise. +package httpguts + +import ( + "net/textproto" + "strings" +) + +// SniffedContentType reports whether ct is a Content-Type that is known +// to cause client-side content sniffing. +// +// This provides just a partial implementation of mime.ParseMediaType +// with the assumption that the Content-Type is not attacker controlled. +func SniffedContentType(ct string) bool { + if i := strings.Index(ct, ";"); i != -1 { + ct = ct[:i] + } + ct = strings.ToLower(strings.TrimSpace(ct)) + return ct == "text/plain" || ct == "application/octet-stream" || + ct == "application/unknown" || ct == "unknown/unknown" || ct == "*/*" || + !strings.Contains(ct, "/") +} + +// ValidTrailerHeader reports whether name is a valid header field name to appear +// in trailers. +// See RFC 7230, Section 4.1.2 +func ValidTrailerHeader(name string) bool { + name = textproto.CanonicalMIMEHeaderKey(name) + if strings.HasPrefix(name, "If-") || badTrailer[name] { + return false + } + return true +} + +var badTrailer = map[string]bool{ + "Authorization": true, + "Cache-Control": true, + "Connection": true, + "Content-Encoding": true, + "Content-Length": true, + "Content-Range": true, + "Content-Type": true, + "Expect": true, + "Host": true, + "Keep-Alive": true, + "Max-Forwards": true, + "Pragma": true, + "Proxy-Authenticate": true, + "Proxy-Authorization": true, + "Proxy-Connection": true, + "Range": true, + "Realm": true, + "Te": true, + "Trailer": true, + "Transfer-Encoding": true, + "Www-Authenticate": true, +} diff --git a/vendor/golang.org/x/net/http/httpguts/httplex.go b/vendor/golang.org/x/net/http/httpguts/httplex.go new file mode 100644 index 00000000000..e7de24ee64e --- /dev/null +++ b/vendor/golang.org/x/net/http/httpguts/httplex.go @@ -0,0 +1,346 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httpguts + +import ( + "net" + "strings" + "unicode/utf8" + + "golang.org/x/net/idna" +) + +var isTokenTable = [127]bool{ + '!': true, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '*': true, + '+': true, + '-': true, + '.': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'W': true, + 'V': true, + 'X': true, + 'Y': true, + 'Z': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '|': true, + '~': true, +} + +func IsTokenRune(r rune) bool { + i := int(r) + return i < len(isTokenTable) && isTokenTable[i] +} + +func isNotToken(r rune) bool { + return !IsTokenRune(r) +} + +// HeaderValuesContainsToken reports whether any string in values +// contains the provided token, ASCII case-insensitively. +func HeaderValuesContainsToken(values []string, token string) bool { + for _, v := range values { + if headerValueContainsToken(v, token) { + return true + } + } + return false +} + +// isOWS reports whether b is an optional whitespace byte, as defined +// by RFC 7230 section 3.2.3. +func isOWS(b byte) bool { return b == ' ' || b == '\t' } + +// trimOWS returns x with all optional whitespace removes from the +// beginning and end. +func trimOWS(x string) string { + // TODO: consider using strings.Trim(x, " \t") instead, + // if and when it's fast enough. See issue 10292. + // But this ASCII-only code will probably always beat UTF-8 + // aware code. + for len(x) > 0 && isOWS(x[0]) { + x = x[1:] + } + for len(x) > 0 && isOWS(x[len(x)-1]) { + x = x[:len(x)-1] + } + return x +} + +// headerValueContainsToken reports whether v (assumed to be a +// 0#element, in the ABNF extension described in RFC 7230 section 7) +// contains token amongst its comma-separated tokens, ASCII +// case-insensitively. +func headerValueContainsToken(v string, token string) bool { + v = trimOWS(v) + if comma := strings.IndexByte(v, ','); comma != -1 { + return tokenEqual(trimOWS(v[:comma]), token) || headerValueContainsToken(v[comma+1:], token) + } + return tokenEqual(v, token) +} + +// lowerASCII returns the ASCII lowercase version of b. +func lowerASCII(b byte) byte { + if 'A' <= b && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} + +// tokenEqual reports whether t1 and t2 are equal, ASCII case-insensitively. +func tokenEqual(t1, t2 string) bool { + if len(t1) != len(t2) { + return false + } + for i, b := range t1 { + if b >= utf8.RuneSelf { + // No UTF-8 or non-ASCII allowed in tokens. + return false + } + if lowerASCII(byte(b)) != lowerASCII(t2[i]) { + return false + } + } + return true +} + +// isLWS reports whether b is linear white space, according +// to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 +// LWS = [CRLF] 1*( SP | HT ) +func isLWS(b byte) bool { return b == ' ' || b == '\t' } + +// isCTL reports whether b is a control byte, according +// to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 +// CTL = +func isCTL(b byte) bool { + const del = 0x7f // a CTL + return b < ' ' || b == del +} + +// ValidHeaderFieldName reports whether v is a valid HTTP/1.x header name. +// HTTP/2 imposes the additional restriction that uppercase ASCII +// letters are not allowed. +// +// RFC 7230 says: +// header-field = field-name ":" OWS field-value OWS +// field-name = token +// token = 1*tchar +// tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / +// "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA +func ValidHeaderFieldName(v string) bool { + if len(v) == 0 { + return false + } + for _, r := range v { + if !IsTokenRune(r) { + return false + } + } + return true +} + +// ValidHostHeader reports whether h is a valid host header. +func ValidHostHeader(h string) bool { + // The latest spec is actually this: + // + // http://tools.ietf.org/html/rfc7230#section-5.4 + // Host = uri-host [ ":" port ] + // + // Where uri-host is: + // http://tools.ietf.org/html/rfc3986#section-3.2.2 + // + // But we're going to be much more lenient for now and just + // search for any byte that's not a valid byte in any of those + // expressions. + for i := 0; i < len(h); i++ { + if !validHostByte[h[i]] { + return false + } + } + return true +} + +// See the validHostHeader comment. +var validHostByte = [256]bool{ + '0': true, '1': true, '2': true, '3': true, '4': true, '5': true, '6': true, '7': true, + '8': true, '9': true, + + 'a': true, 'b': true, 'c': true, 'd': true, 'e': true, 'f': true, 'g': true, 'h': true, + 'i': true, 'j': true, 'k': true, 'l': true, 'm': true, 'n': true, 'o': true, 'p': true, + 'q': true, 'r': true, 's': true, 't': true, 'u': true, 'v': true, 'w': true, 'x': true, + 'y': true, 'z': true, + + 'A': true, 'B': true, 'C': true, 'D': true, 'E': true, 'F': true, 'G': true, 'H': true, + 'I': true, 'J': true, 'K': true, 'L': true, 'M': true, 'N': true, 'O': true, 'P': true, + 'Q': true, 'R': true, 'S': true, 'T': true, 'U': true, 'V': true, 'W': true, 'X': true, + 'Y': true, 'Z': true, + + '!': true, // sub-delims + '$': true, // sub-delims + '%': true, // pct-encoded (and used in IPv6 zones) + '&': true, // sub-delims + '(': true, // sub-delims + ')': true, // sub-delims + '*': true, // sub-delims + '+': true, // sub-delims + ',': true, // sub-delims + '-': true, // unreserved + '.': true, // unreserved + ':': true, // IPv6address + Host expression's optional port + ';': true, // sub-delims + '=': true, // sub-delims + '[': true, + '\'': true, // sub-delims + ']': true, + '_': true, // unreserved + '~': true, // unreserved +} + +// ValidHeaderFieldValue reports whether v is a valid "field-value" according to +// http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 : +// +// message-header = field-name ":" [ field-value ] +// field-value = *( field-content | LWS ) +// field-content = +// +// http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 : +// +// TEXT = +// LWS = [CRLF] 1*( SP | HT ) +// CTL = +// +// RFC 7230 says: +// field-value = *( field-content / obs-fold ) +// obj-fold = N/A to http2, and deprecated +// field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] +// field-vchar = VCHAR / obs-text +// obs-text = %x80-FF +// VCHAR = "any visible [USASCII] character" +// +// http2 further says: "Similarly, HTTP/2 allows header field values +// that are not valid. While most of the values that can be encoded +// will not alter header field parsing, carriage return (CR, ASCII +// 0xd), line feed (LF, ASCII 0xa), and the zero character (NUL, ASCII +// 0x0) might be exploited by an attacker if they are translated +// verbatim. Any request or response that contains a character not +// permitted in a header field value MUST be treated as malformed +// (Section 8.1.2.6). Valid characters are defined by the +// field-content ABNF rule in Section 3.2 of [RFC7230]." +// +// This function does not (yet?) properly handle the rejection of +// strings that begin or end with SP or HTAB. +func ValidHeaderFieldValue(v string) bool { + for i := 0; i < len(v); i++ { + b := v[i] + if isCTL(b) && !isLWS(b) { + return false + } + } + return true +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +// PunycodeHostPort returns the IDNA Punycode version +// of the provided "host" or "host:port" string. +func PunycodeHostPort(v string) (string, error) { + if isASCII(v) { + return v, nil + } + + host, port, err := net.SplitHostPort(v) + if err != nil { + // The input 'v' argument was just a "host" argument, + // without a port. This error should not be returned + // to the caller. + host = v + port = "" + } + host, err = idna.ToASCII(host) + if err != nil { + // Non-UTF-8? Not representable in Punycode, in any + // case. + return "", err + } + if port == "" { + return host, nil + } + return net.JoinHostPort(host, port), nil +} diff --git a/vendor/golang.org/x/text/secure/bidirule/bidirule.go b/vendor/golang.org/x/text/secure/bidirule/bidirule.go new file mode 100644 index 00000000000..a7161bdd9b6 --- /dev/null +++ b/vendor/golang.org/x/text/secure/bidirule/bidirule.go @@ -0,0 +1,342 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bidirule implements the Bidi Rule defined by RFC 5893. +// +// This package is under development. The API may change without notice and +// without preserving backward compatibility. +package bidirule + +import ( + "errors" + "unicode/utf8" + + "golang.org/x/text/transform" + "golang.org/x/text/unicode/bidi" +) + +// This file contains an implementation of RFC 5893: Right-to-Left Scripts for +// Internationalized Domain Names for Applications (IDNA) +// +// A label is an individual component of a domain name. Labels are usually +// shown separated by dots; for example, the domain name "www.example.com" is +// composed of three labels: "www", "example", and "com". +// +// An RTL label is a label that contains at least one character of class R, AL, +// or AN. An LTR label is any label that is not an RTL label. +// +// A "Bidi domain name" is a domain name that contains at least one RTL label. +// +// The following guarantees can be made based on the above: +// +// o In a domain name consisting of only labels that satisfy the rule, +// the requirements of Section 3 are satisfied. Note that even LTR +// labels and pure ASCII labels have to be tested. +// +// o In a domain name consisting of only LDH labels (as defined in the +// Definitions document [RFC5890]) and labels that satisfy the rule, +// the requirements of Section 3 are satisfied as long as a label +// that starts with an ASCII digit does not come after a +// right-to-left label. +// +// No guarantee is given for other combinations. + +// ErrInvalid indicates a label is invalid according to the Bidi Rule. +var ErrInvalid = errors.New("bidirule: failed Bidi Rule") + +type ruleState uint8 + +const ( + ruleInitial ruleState = iota + ruleLTR + ruleLTRFinal + ruleRTL + ruleRTLFinal + ruleInvalid +) + +type ruleTransition struct { + next ruleState + mask uint16 +} + +var transitions = [...][2]ruleTransition{ + // [2.1] The first character must be a character with Bidi property L, R, or + // AL. If it has the R or AL property, it is an RTL label; if it has the L + // property, it is an LTR label. + ruleInitial: { + {ruleLTRFinal, 1 << bidi.L}, + {ruleRTLFinal, 1< 0; count-- { + p.openers.Remove(p.openers.Front()) + } + break + } + } + sort.Sort(p.pairPositions) + // if we get here, the closing bracket matched no openers + // and gets ignored + } + } +} + +// Bracket pairs within an isolating run sequence are processed as units so +// that both the opening and the closing paired bracket in a pair resolve to +// the same direction. +// +// N0. Process bracket pairs in an isolating run sequence sequentially in +// the logical order of the text positions of the opening paired brackets +// using the logic given below. Within this scope, bidirectional types EN +// and AN are treated as R. +// +// Identify the bracket pairs in the current isolating run sequence +// according to BD16. For each bracket-pair element in the list of pairs of +// text positions: +// +// a Inspect the bidirectional types of the characters enclosed within the +// bracket pair. +// +// b If any strong type (either L or R) matching the embedding direction is +// found, set the type for both brackets in the pair to match the embedding +// direction. +// +// o [ e ] o -> o e e e o +// +// o [ o e ] -> o e o e e +// +// o [ NI e ] -> o e NI e e +// +// c Otherwise, if a strong type (opposite the embedding direction) is +// found, test for adjacent strong types as follows: 1 First, check +// backwards before the opening paired bracket until the first strong type +// (L, R, or sos) is found. If that first preceding strong type is opposite +// the embedding direction, then set the type for both brackets in the pair +// to that type. 2 Otherwise, set the type for both brackets in the pair to +// the embedding direction. +// +// o [ o ] e -> o o o o e +// +// o [ o NI ] o -> o o o NI o o +// +// e [ o ] o -> e e o e o +// +// e [ o ] e -> e e o e e +// +// e ( o [ o ] NI ) e -> e e o o o o NI e e +// +// d Otherwise, do not set the type for the current bracket pair. Note that +// if the enclosed text contains no strong types the paired brackets will +// both resolve to the same level when resolved individually using rules N1 +// and N2. +// +// e ( NI ) o -> e ( NI ) o + +// getStrongTypeN0 maps character's directional code to strong type as required +// by rule N0. +// +// TODO: have separate type for "strong" directionality. +func (p *bracketPairer) getStrongTypeN0(index int) Class { + switch p.codesIsolatedRun[index] { + // in the scope of N0, number types are treated as R + case EN, AN, AL, R: + return R + case L: + return L + default: + return ON + } +} + +// classifyPairContent reports the strong types contained inside a Bracket Pair, +// assuming the given embedding direction. +// +// It returns ON if no strong type is found. If a single strong type is found, +// it returns this this type. Otherwise it returns the embedding direction. +// +// TODO: use separate type for "strong" directionality. +func (p *bracketPairer) classifyPairContent(loc bracketPair, dirEmbed Class) Class { + dirOpposite := ON + for i := loc.opener + 1; i < loc.closer; i++ { + dir := p.getStrongTypeN0(i) + if dir == ON { + continue + } + if dir == dirEmbed { + return dir // type matching embedding direction found + } + dirOpposite = dir + } + // return ON if no strong type found, or class opposite to dirEmbed + return dirOpposite +} + +// classBeforePair determines which strong types are present before a Bracket +// Pair. Return R or L if strong type found, otherwise ON. +func (p *bracketPairer) classBeforePair(loc bracketPair) Class { + for i := loc.opener - 1; i >= 0; i-- { + if dir := p.getStrongTypeN0(i); dir != ON { + return dir + } + } + // no strong types found, return sos + return p.sos +} + +// assignBracketType implements rule N0 for a single bracket pair. +func (p *bracketPairer) assignBracketType(loc bracketPair, dirEmbed Class, initialTypes []Class) { + // rule "N0, a", inspect contents of pair + dirPair := p.classifyPairContent(loc, dirEmbed) + + // dirPair is now L, R, or N (no strong type found) + + // the following logical tests are performed out of order compared to + // the statement of the rules but yield the same results + if dirPair == ON { + return // case "d" - nothing to do + } + + if dirPair != dirEmbed { + // case "c": strong type found, opposite - check before (c.1) + dirPair = p.classBeforePair(loc) + if dirPair == dirEmbed || dirPair == ON { + // no strong opposite type found before - use embedding (c.2) + dirPair = dirEmbed + } + } + // else: case "b", strong type found matching embedding, + // no explicit action needed, as dirPair is already set to embedding + // direction + + // set the bracket types to the type found + p.setBracketsToType(loc, dirPair, initialTypes) +} + +func (p *bracketPairer) setBracketsToType(loc bracketPair, dirPair Class, initialTypes []Class) { + p.codesIsolatedRun[loc.opener] = dirPair + p.codesIsolatedRun[loc.closer] = dirPair + + for i := loc.opener + 1; i < loc.closer; i++ { + index := p.indexes[i] + if initialTypes[index] != NSM { + break + } + p.codesIsolatedRun[i] = dirPair + } + + for i := loc.closer + 1; i < len(p.indexes); i++ { + index := p.indexes[i] + if initialTypes[index] != NSM { + break + } + p.codesIsolatedRun[i] = dirPair + } +} + +// resolveBrackets implements rule N0 for a list of pairs. +func (p *bracketPairer) resolveBrackets(dirEmbed Class, initialTypes []Class) { + for _, loc := range p.pairPositions { + p.assignBracketType(loc, dirEmbed, initialTypes) + } +} diff --git a/vendor/golang.org/x/text/unicode/bidi/core.go b/vendor/golang.org/x/text/unicode/bidi/core.go new file mode 100644 index 00000000000..ec52f144807 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/core.go @@ -0,0 +1,1058 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bidi + +import "log" + +// This implementation is a port based on the reference implementation found at: +// http://www.unicode.org/Public/PROGRAMS/BidiReferenceJava/ +// +// described in Unicode Bidirectional Algorithm (UAX #9). +// +// Input: +// There are two levels of input to the algorithm, since clients may prefer to +// supply some information from out-of-band sources rather than relying on the +// default behavior. +// +// - Bidi class array +// - Bidi class array, with externally supplied base line direction +// +// Output: +// Output is separated into several stages: +// +// - levels array over entire paragraph +// - reordering array over entire paragraph +// - levels array over line +// - reordering array over line +// +// Note that for conformance to the Unicode Bidirectional Algorithm, +// implementations are only required to generate correct reordering and +// character directionality (odd or even levels) over a line. Generating +// identical level arrays over a line is not required. Bidi explicit format +// codes (LRE, RLE, LRO, RLO, PDF) and BN can be assigned arbitrary levels and +// positions as long as the rest of the input is properly reordered. +// +// As the algorithm is defined to operate on a single paragraph at a time, this +// implementation is written to handle single paragraphs. Thus rule P1 is +// presumed by this implementation-- the data provided to the implementation is +// assumed to be a single paragraph, and either contains no 'B' codes, or a +// single 'B' code at the end of the input. 'B' is allowed as input to +// illustrate how the algorithm assigns it a level. +// +// Also note that rules L3 and L4 depend on the rendering engine that uses the +// result of the bidi algorithm. This implementation assumes that the rendering +// engine expects combining marks in visual order (e.g. to the left of their +// base character in RTL runs) and that it adjusts the glyphs used to render +// mirrored characters that are in RTL runs so that they render appropriately. + +// level is the embedding level of a character. Even embedding levels indicate +// left-to-right order and odd levels indicate right-to-left order. The special +// level of -1 is reserved for undefined order. +type level int8 + +const implicitLevel level = -1 + +// in returns if x is equal to any of the values in set. +func (c Class) in(set ...Class) bool { + for _, s := range set { + if c == s { + return true + } + } + return false +} + +// A paragraph contains the state of a paragraph. +type paragraph struct { + initialTypes []Class + + // Arrays of properties needed for paired bracket evaluation in N0 + pairTypes []bracketType // paired Bracket types for paragraph + pairValues []rune // rune for opening bracket or pbOpen and pbClose; 0 for pbNone + + embeddingLevel level // default: = implicitLevel; + + // at the paragraph levels + resultTypes []Class + resultLevels []level + + // Index of matching PDI for isolate initiator characters. For other + // characters, the value of matchingPDI will be set to -1. For isolate + // initiators with no matching PDI, matchingPDI will be set to the length of + // the input string. + matchingPDI []int + + // Index of matching isolate initiator for PDI characters. For other + // characters, and for PDIs with no matching isolate initiator, the value of + // matchingIsolateInitiator will be set to -1. + matchingIsolateInitiator []int +} + +// newParagraph initializes a paragraph. The user needs to supply a few arrays +// corresponding to the preprocessed text input. The types correspond to the +// Unicode BiDi classes for each rune. pairTypes indicates the bracket type for +// each rune. pairValues provides a unique bracket class identifier for each +// rune (suggested is the rune of the open bracket for opening and matching +// close brackets, after normalization). The embedding levels are optional, but +// may be supplied to encode embedding levels of styled text. +// +// TODO: return an error. +func newParagraph(types []Class, pairTypes []bracketType, pairValues []rune, levels level) *paragraph { + validateTypes(types) + validatePbTypes(pairTypes) + validatePbValues(pairValues, pairTypes) + validateParagraphEmbeddingLevel(levels) + + p := ¶graph{ + initialTypes: append([]Class(nil), types...), + embeddingLevel: levels, + + pairTypes: pairTypes, + pairValues: pairValues, + + resultTypes: append([]Class(nil), types...), + } + p.run() + return p +} + +func (p *paragraph) Len() int { return len(p.initialTypes) } + +// The algorithm. Does not include line-based processing (Rules L1, L2). +// These are applied later in the line-based phase of the algorithm. +func (p *paragraph) run() { + p.determineMatchingIsolates() + + // 1) determining the paragraph level + // Rule P1 is the requirement for entering this algorithm. + // Rules P2, P3. + // If no externally supplied paragraph embedding level, use default. + if p.embeddingLevel == implicitLevel { + p.embeddingLevel = p.determineParagraphEmbeddingLevel(0, p.Len()) + } + + // Initialize result levels to paragraph embedding level. + p.resultLevels = make([]level, p.Len()) + setLevels(p.resultLevels, p.embeddingLevel) + + // 2) Explicit levels and directions + // Rules X1-X8. + p.determineExplicitEmbeddingLevels() + + // Rule X9. + // We do not remove the embeddings, the overrides, the PDFs, and the BNs + // from the string explicitly. But they are not copied into isolating run + // sequences when they are created, so they are removed for all + // practical purposes. + + // Rule X10. + // Run remainder of algorithm one isolating run sequence at a time + for _, seq := range p.determineIsolatingRunSequences() { + // 3) resolving weak types + // Rules W1-W7. + seq.resolveWeakTypes() + + // 4a) resolving paired brackets + // Rule N0 + resolvePairedBrackets(seq) + + // 4b) resolving neutral types + // Rules N1-N3. + seq.resolveNeutralTypes() + + // 5) resolving implicit embedding levels + // Rules I1, I2. + seq.resolveImplicitLevels() + + // Apply the computed levels and types + seq.applyLevelsAndTypes() + } + + // Assign appropriate levels to 'hide' LREs, RLEs, LROs, RLOs, PDFs, and + // BNs. This is for convenience, so the resulting level array will have + // a value for every character. + p.assignLevelsToCharactersRemovedByX9() +} + +// determineMatchingIsolates determines the matching PDI for each isolate +// initiator and vice versa. +// +// Definition BD9. +// +// At the end of this function: +// +// - The member variable matchingPDI is set to point to the index of the +// matching PDI character for each isolate initiator character. If there is +// no matching PDI, it is set to the length of the input text. For other +// characters, it is set to -1. +// - The member variable matchingIsolateInitiator is set to point to the +// index of the matching isolate initiator character for each PDI character. +// If there is no matching isolate initiator, or the character is not a PDI, +// it is set to -1. +func (p *paragraph) determineMatchingIsolates() { + p.matchingPDI = make([]int, p.Len()) + p.matchingIsolateInitiator = make([]int, p.Len()) + + for i := range p.matchingIsolateInitiator { + p.matchingIsolateInitiator[i] = -1 + } + + for i := range p.matchingPDI { + p.matchingPDI[i] = -1 + + if t := p.resultTypes[i]; t.in(LRI, RLI, FSI) { + depthCounter := 1 + for j := i + 1; j < p.Len(); j++ { + if u := p.resultTypes[j]; u.in(LRI, RLI, FSI) { + depthCounter++ + } else if u == PDI { + if depthCounter--; depthCounter == 0 { + p.matchingPDI[i] = j + p.matchingIsolateInitiator[j] = i + break + } + } + } + if p.matchingPDI[i] == -1 { + p.matchingPDI[i] = p.Len() + } + } + } +} + +// determineParagraphEmbeddingLevel reports the resolved paragraph direction of +// the substring limited by the given range [start, end). +// +// Determines the paragraph level based on rules P2, P3. This is also used +// in rule X5c to find if an FSI should resolve to LRI or RLI. +func (p *paragraph) determineParagraphEmbeddingLevel(start, end int) level { + var strongType Class = unknownClass + + // Rule P2. + for i := start; i < end; i++ { + if t := p.resultTypes[i]; t.in(L, AL, R) { + strongType = t + break + } else if t.in(FSI, LRI, RLI) { + i = p.matchingPDI[i] // skip over to the matching PDI + if i > end { + log.Panic("assert (i <= end)") + } + } + } + // Rule P3. + switch strongType { + case unknownClass: // none found + // default embedding level when no strong types found is 0. + return 0 + case L: + return 0 + default: // AL, R + return 1 + } +} + +const maxDepth = 125 + +// This stack will store the embedding levels and override and isolated +// statuses +type directionalStatusStack struct { + stackCounter int + embeddingLevelStack [maxDepth + 1]level + overrideStatusStack [maxDepth + 1]Class + isolateStatusStack [maxDepth + 1]bool +} + +func (s *directionalStatusStack) empty() { s.stackCounter = 0 } +func (s *directionalStatusStack) pop() { s.stackCounter-- } +func (s *directionalStatusStack) depth() int { return s.stackCounter } + +func (s *directionalStatusStack) push(level level, overrideStatus Class, isolateStatus bool) { + s.embeddingLevelStack[s.stackCounter] = level + s.overrideStatusStack[s.stackCounter] = overrideStatus + s.isolateStatusStack[s.stackCounter] = isolateStatus + s.stackCounter++ +} + +func (s *directionalStatusStack) lastEmbeddingLevel() level { + return s.embeddingLevelStack[s.stackCounter-1] +} + +func (s *directionalStatusStack) lastDirectionalOverrideStatus() Class { + return s.overrideStatusStack[s.stackCounter-1] +} + +func (s *directionalStatusStack) lastDirectionalIsolateStatus() bool { + return s.isolateStatusStack[s.stackCounter-1] +} + +// Determine explicit levels using rules X1 - X8 +func (p *paragraph) determineExplicitEmbeddingLevels() { + var stack directionalStatusStack + var overflowIsolateCount, overflowEmbeddingCount, validIsolateCount int + + // Rule X1. + stack.push(p.embeddingLevel, ON, false) + + for i, t := range p.resultTypes { + // Rules X2, X3, X4, X5, X5a, X5b, X5c + switch t { + case RLE, LRE, RLO, LRO, RLI, LRI, FSI: + isIsolate := t.in(RLI, LRI, FSI) + isRTL := t.in(RLE, RLO, RLI) + + // override if this is an FSI that resolves to RLI + if t == FSI { + isRTL = (p.determineParagraphEmbeddingLevel(i+1, p.matchingPDI[i]) == 1) + } + if isIsolate { + p.resultLevels[i] = stack.lastEmbeddingLevel() + if stack.lastDirectionalOverrideStatus() != ON { + p.resultTypes[i] = stack.lastDirectionalOverrideStatus() + } + } + + var newLevel level + if isRTL { + // least greater odd + newLevel = (stack.lastEmbeddingLevel() + 1) | 1 + } else { + // least greater even + newLevel = (stack.lastEmbeddingLevel() + 2) &^ 1 + } + + if newLevel <= maxDepth && overflowIsolateCount == 0 && overflowEmbeddingCount == 0 { + if isIsolate { + validIsolateCount++ + } + // Push new embedding level, override status, and isolated + // status. + // No check for valid stack counter, since the level check + // suffices. + switch t { + case LRO: + stack.push(newLevel, L, isIsolate) + case RLO: + stack.push(newLevel, R, isIsolate) + default: + stack.push(newLevel, ON, isIsolate) + } + // Not really part of the spec + if !isIsolate { + p.resultLevels[i] = newLevel + } + } else { + // This is an invalid explicit formatting character, + // so apply the "Otherwise" part of rules X2-X5b. + if isIsolate { + overflowIsolateCount++ + } else { // !isIsolate + if overflowIsolateCount == 0 { + overflowEmbeddingCount++ + } + } + } + + // Rule X6a + case PDI: + if overflowIsolateCount > 0 { + overflowIsolateCount-- + } else if validIsolateCount == 0 { + // do nothing + } else { + overflowEmbeddingCount = 0 + for !stack.lastDirectionalIsolateStatus() { + stack.pop() + } + stack.pop() + validIsolateCount-- + } + p.resultLevels[i] = stack.lastEmbeddingLevel() + + // Rule X7 + case PDF: + // Not really part of the spec + p.resultLevels[i] = stack.lastEmbeddingLevel() + + if overflowIsolateCount > 0 { + // do nothing + } else if overflowEmbeddingCount > 0 { + overflowEmbeddingCount-- + } else if !stack.lastDirectionalIsolateStatus() && stack.depth() >= 2 { + stack.pop() + } + + case B: // paragraph separator. + // Rule X8. + + // These values are reset for clarity, in this implementation B + // can only occur as the last code in the array. + stack.empty() + overflowIsolateCount = 0 + overflowEmbeddingCount = 0 + validIsolateCount = 0 + p.resultLevels[i] = p.embeddingLevel + + default: + p.resultLevels[i] = stack.lastEmbeddingLevel() + if stack.lastDirectionalOverrideStatus() != ON { + p.resultTypes[i] = stack.lastDirectionalOverrideStatus() + } + } + } +} + +type isolatingRunSequence struct { + p *paragraph + + indexes []int // indexes to the original string + + types []Class // type of each character using the index + resolvedLevels []level // resolved levels after application of rules + level level + sos, eos Class +} + +func (i *isolatingRunSequence) Len() int { return len(i.indexes) } + +func maxLevel(a, b level) level { + if a > b { + return a + } + return b +} + +// Rule X10, second bullet: Determine the start-of-sequence (sos) and end-of-sequence (eos) types, +// either L or R, for each isolating run sequence. +func (p *paragraph) isolatingRunSequence(indexes []int) *isolatingRunSequence { + length := len(indexes) + types := make([]Class, length) + for i, x := range indexes { + types[i] = p.resultTypes[x] + } + + // assign level, sos and eos + prevChar := indexes[0] - 1 + for prevChar >= 0 && isRemovedByX9(p.initialTypes[prevChar]) { + prevChar-- + } + prevLevel := p.embeddingLevel + if prevChar >= 0 { + prevLevel = p.resultLevels[prevChar] + } + + var succLevel level + lastType := types[length-1] + if lastType.in(LRI, RLI, FSI) { + succLevel = p.embeddingLevel + } else { + // the first character after the end of run sequence + limit := indexes[length-1] + 1 + for ; limit < p.Len() && isRemovedByX9(p.initialTypes[limit]); limit++ { + + } + succLevel = p.embeddingLevel + if limit < p.Len() { + succLevel = p.resultLevels[limit] + } + } + level := p.resultLevels[indexes[0]] + return &isolatingRunSequence{ + p: p, + indexes: indexes, + types: types, + level: level, + sos: typeForLevel(maxLevel(prevLevel, level)), + eos: typeForLevel(maxLevel(succLevel, level)), + } +} + +// Resolving weak types Rules W1-W7. +// +// Note that some weak types (EN, AN) remain after this processing is +// complete. +func (s *isolatingRunSequence) resolveWeakTypes() { + + // on entry, only these types remain + s.assertOnly(L, R, AL, EN, ES, ET, AN, CS, B, S, WS, ON, NSM, LRI, RLI, FSI, PDI) + + // Rule W1. + // Changes all NSMs. + preceedingCharacterType := s.sos + for i, t := range s.types { + if t == NSM { + s.types[i] = preceedingCharacterType + } else { + if t.in(LRI, RLI, FSI, PDI) { + preceedingCharacterType = ON + } + preceedingCharacterType = t + } + } + + // Rule W2. + // EN does not change at the start of the run, because sos != AL. + for i, t := range s.types { + if t == EN { + for j := i - 1; j >= 0; j-- { + if t := s.types[j]; t.in(L, R, AL) { + if t == AL { + s.types[i] = AN + } + break + } + } + } + } + + // Rule W3. + for i, t := range s.types { + if t == AL { + s.types[i] = R + } + } + + // Rule W4. + // Since there must be values on both sides for this rule to have an + // effect, the scan skips the first and last value. + // + // Although the scan proceeds left to right, and changes the type + // values in a way that would appear to affect the computations + // later in the scan, there is actually no problem. A change in the + // current value can only affect the value to its immediate right, + // and only affect it if it is ES or CS. But the current value can + // only change if the value to its right is not ES or CS. Thus + // either the current value will not change, or its change will have + // no effect on the remainder of the analysis. + + for i := 1; i < s.Len()-1; i++ { + t := s.types[i] + if t == ES || t == CS { + prevSepType := s.types[i-1] + succSepType := s.types[i+1] + if prevSepType == EN && succSepType == EN { + s.types[i] = EN + } else if s.types[i] == CS && prevSepType == AN && succSepType == AN { + s.types[i] = AN + } + } + } + + // Rule W5. + for i, t := range s.types { + if t == ET { + // locate end of sequence + runStart := i + runEnd := s.findRunLimit(runStart, ET) + + // check values at ends of sequence + t := s.sos + if runStart > 0 { + t = s.types[runStart-1] + } + if t != EN { + t = s.eos + if runEnd < len(s.types) { + t = s.types[runEnd] + } + } + if t == EN { + setTypes(s.types[runStart:runEnd], EN) + } + // continue at end of sequence + i = runEnd + } + } + + // Rule W6. + for i, t := range s.types { + if t.in(ES, ET, CS) { + s.types[i] = ON + } + } + + // Rule W7. + for i, t := range s.types { + if t == EN { + // set default if we reach start of run + prevStrongType := s.sos + for j := i - 1; j >= 0; j-- { + t = s.types[j] + if t == L || t == R { // AL's have been changed to R + prevStrongType = t + break + } + } + if prevStrongType == L { + s.types[i] = L + } + } + } +} + +// 6) resolving neutral types Rules N1-N2. +func (s *isolatingRunSequence) resolveNeutralTypes() { + + // on entry, only these types can be in resultTypes + s.assertOnly(L, R, EN, AN, B, S, WS, ON, RLI, LRI, FSI, PDI) + + for i, t := range s.types { + switch t { + case WS, ON, B, S, RLI, LRI, FSI, PDI: + // find bounds of run of neutrals + runStart := i + runEnd := s.findRunLimit(runStart, B, S, WS, ON, RLI, LRI, FSI, PDI) + + // determine effective types at ends of run + var leadType, trailType Class + + // Note that the character found can only be L, R, AN, or + // EN. + if runStart == 0 { + leadType = s.sos + } else { + leadType = s.types[runStart-1] + if leadType.in(AN, EN) { + leadType = R + } + } + if runEnd == len(s.types) { + trailType = s.eos + } else { + trailType = s.types[runEnd] + if trailType.in(AN, EN) { + trailType = R + } + } + + var resolvedType Class + if leadType == trailType { + // Rule N1. + resolvedType = leadType + } else { + // Rule N2. + // Notice the embedding level of the run is used, not + // the paragraph embedding level. + resolvedType = typeForLevel(s.level) + } + + setTypes(s.types[runStart:runEnd], resolvedType) + + // skip over run of (former) neutrals + i = runEnd + } + } +} + +func setLevels(levels []level, newLevel level) { + for i := range levels { + levels[i] = newLevel + } +} + +func setTypes(types []Class, newType Class) { + for i := range types { + types[i] = newType + } +} + +// 7) resolving implicit embedding levels Rules I1, I2. +func (s *isolatingRunSequence) resolveImplicitLevels() { + + // on entry, only these types can be in resultTypes + s.assertOnly(L, R, EN, AN) + + s.resolvedLevels = make([]level, len(s.types)) + setLevels(s.resolvedLevels, s.level) + + if (s.level & 1) == 0 { // even level + for i, t := range s.types { + // Rule I1. + if t == L { + // no change + } else if t == R { + s.resolvedLevels[i] += 1 + } else { // t == AN || t == EN + s.resolvedLevels[i] += 2 + } + } + } else { // odd level + for i, t := range s.types { + // Rule I2. + if t == R { + // no change + } else { // t == L || t == AN || t == EN + s.resolvedLevels[i] += 1 + } + } + } +} + +// Applies the levels and types resolved in rules W1-I2 to the +// resultLevels array. +func (s *isolatingRunSequence) applyLevelsAndTypes() { + for i, x := range s.indexes { + s.p.resultTypes[x] = s.types[i] + s.p.resultLevels[x] = s.resolvedLevels[i] + } +} + +// Return the limit of the run consisting only of the types in validSet +// starting at index. This checks the value at index, and will return +// index if that value is not in validSet. +func (s *isolatingRunSequence) findRunLimit(index int, validSet ...Class) int { +loop: + for ; index < len(s.types); index++ { + t := s.types[index] + for _, valid := range validSet { + if t == valid { + continue loop + } + } + return index // didn't find a match in validSet + } + return len(s.types) +} + +// Algorithm validation. Assert that all values in types are in the +// provided set. +func (s *isolatingRunSequence) assertOnly(codes ...Class) { +loop: + for i, t := range s.types { + for _, c := range codes { + if t == c { + continue loop + } + } + log.Panicf("invalid bidi code %s present in assertOnly at position %d", t, s.indexes[i]) + } +} + +// determineLevelRuns returns an array of level runs. Each level run is +// described as an array of indexes into the input string. +// +// Determines the level runs. Rule X9 will be applied in determining the +// runs, in the way that makes sure the characters that are supposed to be +// removed are not included in the runs. +func (p *paragraph) determineLevelRuns() [][]int { + run := []int{} + allRuns := [][]int{} + currentLevel := implicitLevel + + for i := range p.initialTypes { + if !isRemovedByX9(p.initialTypes[i]) { + if p.resultLevels[i] != currentLevel { + // we just encountered a new run; wrap up last run + if currentLevel >= 0 { // only wrap it up if there was a run + allRuns = append(allRuns, run) + run = nil + } + // Start new run + currentLevel = p.resultLevels[i] + } + run = append(run, i) + } + } + // Wrap up the final run, if any + if len(run) > 0 { + allRuns = append(allRuns, run) + } + return allRuns +} + +// Definition BD13. Determine isolating run sequences. +func (p *paragraph) determineIsolatingRunSequences() []*isolatingRunSequence { + levelRuns := p.determineLevelRuns() + + // Compute the run that each character belongs to + runForCharacter := make([]int, p.Len()) + for i, run := range levelRuns { + for _, index := range run { + runForCharacter[index] = i + } + } + + sequences := []*isolatingRunSequence{} + + var currentRunSequence []int + + for _, run := range levelRuns { + first := run[0] + if p.initialTypes[first] != PDI || p.matchingIsolateInitiator[first] == -1 { + currentRunSequence = nil + // int run = i; + for { + // Copy this level run into currentRunSequence + currentRunSequence = append(currentRunSequence, run...) + + last := currentRunSequence[len(currentRunSequence)-1] + lastT := p.initialTypes[last] + if lastT.in(LRI, RLI, FSI) && p.matchingPDI[last] != p.Len() { + run = levelRuns[runForCharacter[p.matchingPDI[last]]] + } else { + break + } + } + sequences = append(sequences, p.isolatingRunSequence(currentRunSequence)) + } + } + return sequences +} + +// Assign level information to characters removed by rule X9. This is for +// ease of relating the level information to the original input data. Note +// that the levels assigned to these codes are arbitrary, they're chosen so +// as to avoid breaking level runs. +func (p *paragraph) assignLevelsToCharactersRemovedByX9() { + for i, t := range p.initialTypes { + if t.in(LRE, RLE, LRO, RLO, PDF, BN) { + p.resultTypes[i] = t + p.resultLevels[i] = -1 + } + } + // now propagate forward the levels information (could have + // propagated backward, the main thing is not to introduce a level + // break where one doesn't already exist). + + if p.resultLevels[0] == -1 { + p.resultLevels[0] = p.embeddingLevel + } + for i := 1; i < len(p.initialTypes); i++ { + if p.resultLevels[i] == -1 { + p.resultLevels[i] = p.resultLevels[i-1] + } + } + // Embedding information is for informational purposes only so need not be + // adjusted. +} + +// +// Output +// + +// getLevels computes levels array breaking lines at offsets in linebreaks. +// Rule L1. +// +// The linebreaks array must include at least one value. The values must be +// in strictly increasing order (no duplicates) between 1 and the length of +// the text, inclusive. The last value must be the length of the text. +func (p *paragraph) getLevels(linebreaks []int) []level { + // Note that since the previous processing has removed all + // P, S, and WS values from resultTypes, the values referred to + // in these rules are the initial types, before any processing + // has been applied (including processing of overrides). + // + // This example implementation has reinserted explicit format codes + // and BN, in order that the levels array correspond to the + // initial text. Their final placement is not normative. + // These codes are treated like WS in this implementation, + // so they don't interrupt sequences of WS. + + validateLineBreaks(linebreaks, p.Len()) + + result := append([]level(nil), p.resultLevels...) + + // don't worry about linebreaks since if there is a break within + // a series of WS values preceding S, the linebreak itself + // causes the reset. + for i, t := range p.initialTypes { + if t.in(B, S) { + // Rule L1, clauses one and two. + result[i] = p.embeddingLevel + + // Rule L1, clause three. + for j := i - 1; j >= 0; j-- { + if isWhitespace(p.initialTypes[j]) { // including format codes + result[j] = p.embeddingLevel + } else { + break + } + } + } + } + + // Rule L1, clause four. + start := 0 + for _, limit := range linebreaks { + for j := limit - 1; j >= start; j-- { + if isWhitespace(p.initialTypes[j]) { // including format codes + result[j] = p.embeddingLevel + } else { + break + } + } + start = limit + } + + return result +} + +// getReordering returns the reordering of lines from a visual index to a +// logical index for line breaks at the given offsets. +// +// Lines are concatenated from left to right. So for example, the fifth +// character from the left on the third line is +// +// getReordering(linebreaks)[linebreaks[1] + 4] +// +// (linebreaks[1] is the position after the last character of the second +// line, which is also the index of the first character on the third line, +// and adding four gets the fifth character from the left). +// +// The linebreaks array must include at least one value. The values must be +// in strictly increasing order (no duplicates) between 1 and the length of +// the text, inclusive. The last value must be the length of the text. +func (p *paragraph) getReordering(linebreaks []int) []int { + validateLineBreaks(linebreaks, p.Len()) + + return computeMultilineReordering(p.getLevels(linebreaks), linebreaks) +} + +// Return multiline reordering array for a given level array. Reordering +// does not occur across a line break. +func computeMultilineReordering(levels []level, linebreaks []int) []int { + result := make([]int, len(levels)) + + start := 0 + for _, limit := range linebreaks { + tempLevels := make([]level, limit-start) + copy(tempLevels, levels[start:]) + + for j, order := range computeReordering(tempLevels) { + result[start+j] = order + start + } + start = limit + } + return result +} + +// Return reordering array for a given level array. This reorders a single +// line. The reordering is a visual to logical map. For example, the +// leftmost char is string.charAt(order[0]). Rule L2. +func computeReordering(levels []level) []int { + result := make([]int, len(levels)) + // initialize order + for i := range result { + result[i] = i + } + + // locate highest level found on line. + // Note the rules say text, but no reordering across line bounds is + // performed, so this is sufficient. + highestLevel := level(0) + lowestOddLevel := level(maxDepth + 2) + for _, level := range levels { + if level > highestLevel { + highestLevel = level + } + if level&1 != 0 && level < lowestOddLevel { + lowestOddLevel = level + } + } + + for level := highestLevel; level >= lowestOddLevel; level-- { + for i := 0; i < len(levels); i++ { + if levels[i] >= level { + // find range of text at or above this level + start := i + limit := i + 1 + for limit < len(levels) && levels[limit] >= level { + limit++ + } + + for j, k := start, limit-1; j < k; j, k = j+1, k-1 { + result[j], result[k] = result[k], result[j] + } + // skip to end of level run + i = limit + } + } + } + + return result +} + +// isWhitespace reports whether the type is considered a whitespace type for the +// line break rules. +func isWhitespace(c Class) bool { + switch c { + case LRE, RLE, LRO, RLO, PDF, LRI, RLI, FSI, PDI, BN, WS: + return true + } + return false +} + +// isRemovedByX9 reports whether the type is one of the types removed in X9. +func isRemovedByX9(c Class) bool { + switch c { + case LRE, RLE, LRO, RLO, PDF, BN: + return true + } + return false +} + +// typeForLevel reports the strong type (L or R) corresponding to the level. +func typeForLevel(level level) Class { + if (level & 0x1) == 0 { + return L + } + return R +} + +// TODO: change validation to not panic + +func validateTypes(types []Class) { + if len(types) == 0 { + log.Panic("types is null") + } + for i, t := range types[:len(types)-1] { + if t == B { + log.Panicf("B type before end of paragraph at index: %d", i) + } + } +} + +func validateParagraphEmbeddingLevel(embeddingLevel level) { + if embeddingLevel != implicitLevel && + embeddingLevel != 0 && + embeddingLevel != 1 { + log.Panicf("illegal paragraph embedding level: %d", embeddingLevel) + } +} + +func validateLineBreaks(linebreaks []int, textLength int) { + prev := 0 + for i, next := range linebreaks { + if next <= prev { + log.Panicf("bad linebreak: %d at index: %d", next, i) + } + prev = next + } + if prev != textLength { + log.Panicf("last linebreak was %d, want %d", prev, textLength) + } +} + +func validatePbTypes(pairTypes []bracketType) { + if len(pairTypes) == 0 { + log.Panic("pairTypes is null") + } + for i, pt := range pairTypes { + switch pt { + case bpNone, bpOpen, bpClose: + default: + log.Panicf("illegal pairType value at %d: %v", i, pairTypes[i]) + } + } +} + +func validatePbValues(pairValues []rune, pairTypes []bracketType) { + if pairValues == nil { + log.Panic("pairValues is null") + } + if len(pairTypes) != len(pairValues) { + log.Panic("pairTypes is different length from pairValues") + } +} diff --git a/vendor/golang.org/x/text/unicode/bidi/gen.go b/vendor/golang.org/x/text/unicode/bidi/gen.go new file mode 100644 index 00000000000..040f3013d58 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/gen.go @@ -0,0 +1,133 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "flag" + "log" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/triegen" + "golang.org/x/text/internal/ucd" +) + +var outputFile = flag.String("out", "tables.go", "output file") + +func main() { + gen.Init() + gen.Repackage("gen_trieval.go", "trieval.go", "bidi") + gen.Repackage("gen_ranges.go", "ranges_test.go", "bidi") + + genTables() +} + +// bidiClass names and codes taken from class "bc" in +// http://www.unicode.org/Public/8.0.0/ucd/PropertyValueAliases.txt +var bidiClass = map[string]Class{ + "AL": AL, // ArabicLetter + "AN": AN, // ArabicNumber + "B": B, // ParagraphSeparator + "BN": BN, // BoundaryNeutral + "CS": CS, // CommonSeparator + "EN": EN, // EuropeanNumber + "ES": ES, // EuropeanSeparator + "ET": ET, // EuropeanTerminator + "L": L, // LeftToRight + "NSM": NSM, // NonspacingMark + "ON": ON, // OtherNeutral + "R": R, // RightToLeft + "S": S, // SegmentSeparator + "WS": WS, // WhiteSpace + + "FSI": Control, + "PDF": Control, + "PDI": Control, + "LRE": Control, + "LRI": Control, + "LRO": Control, + "RLE": Control, + "RLI": Control, + "RLO": Control, +} + +func genTables() { + if numClass > 0x0F { + log.Fatalf("Too many Class constants (%#x > 0x0F).", numClass) + } + w := gen.NewCodeWriter() + defer w.WriteGoFile(*outputFile, "bidi") + + gen.WriteUnicodeVersion(w) + + t := triegen.NewTrie("bidi") + + // Build data about bracket mapping. These bits need to be or-ed with + // any other bits. + orMask := map[rune]uint64{} + + xorMap := map[rune]int{} + xorMasks := []rune{0} // First value is no-op. + + ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) { + r1 := p.Rune(0) + r2 := p.Rune(1) + xor := r1 ^ r2 + if _, ok := xorMap[xor]; !ok { + xorMap[xor] = len(xorMasks) + xorMasks = append(xorMasks, xor) + } + entry := uint64(xorMap[xor]) << xorMaskShift + switch p.String(2) { + case "o": + entry |= openMask + case "c", "n": + default: + log.Fatalf("Unknown bracket class %q.", p.String(2)) + } + orMask[r1] = entry + }) + + w.WriteComment(` + xorMasks contains masks to be xor-ed with brackets to get the reverse + version.`) + w.WriteVar("xorMasks", xorMasks) + + done := map[rune]bool{} + + insert := func(r rune, c Class) { + if !done[r] { + t.Insert(r, orMask[r]|uint64(c)) + done[r] = true + } + } + + // Insert the derived BiDi properties. + ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) { + r := p.Rune(0) + class, ok := bidiClass[p.String(1)] + if !ok { + log.Fatalf("%U: Unknown BiDi class %q", r, p.String(1)) + } + insert(r, class) + }) + visitDefaults(insert) + + // TODO: use sparse blocks. This would reduce table size considerably + // from the looks of it. + + sz, err := t.Gen(w) + if err != nil { + log.Fatal(err) + } + w.Size += sz +} + +// dummy values to make methods in gen_common compile. The real versions +// will be generated by this file to tables.go. +var ( + xorMasks []rune +) diff --git a/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go b/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go new file mode 100644 index 00000000000..51bd68fa7f3 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go @@ -0,0 +1,57 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "unicode" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/ucd" + "golang.org/x/text/unicode/rangetable" +) + +// These tables are hand-extracted from: +// http://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedBidiClass.txt +func visitDefaults(fn func(r rune, c Class)) { + // first write default values for ranges listed above. + visitRunes(fn, AL, []rune{ + 0x0600, 0x07BF, // Arabic + 0x08A0, 0x08FF, // Arabic Extended-A + 0xFB50, 0xFDCF, // Arabic Presentation Forms + 0xFDF0, 0xFDFF, + 0xFE70, 0xFEFF, + 0x0001EE00, 0x0001EEFF, // Arabic Mathematical Alpha Symbols + }) + visitRunes(fn, R, []rune{ + 0x0590, 0x05FF, // Hebrew + 0x07C0, 0x089F, // Nko et al. + 0xFB1D, 0xFB4F, + 0x00010800, 0x00010FFF, // Cypriot Syllabary et. al. + 0x0001E800, 0x0001EDFF, + 0x0001EF00, 0x0001EFFF, + }) + visitRunes(fn, ET, []rune{ // European Terminator + 0x20A0, 0x20Cf, // Currency symbols + }) + rangetable.Visit(unicode.Noncharacter_Code_Point, func(r rune) { + fn(r, BN) // Boundary Neutral + }) + ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) { + if p.String(1) == "Default_Ignorable_Code_Point" { + fn(p.Rune(0), BN) // Boundary Neutral + } + }) +} + +func visitRunes(fn func(r rune, c Class), c Class, runes []rune) { + for i := 0; i < len(runes); i += 2 { + lo, hi := runes[i], runes[i+1] + for j := lo; j <= hi; j++ { + fn(j, c) + } + } +} diff --git a/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go b/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go new file mode 100644 index 00000000000..9cb99428949 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go @@ -0,0 +1,64 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// Class is the Unicode BiDi class. Each rune has a single class. +type Class uint + +const ( + L Class = iota // LeftToRight + R // RightToLeft + EN // EuropeanNumber + ES // EuropeanSeparator + ET // EuropeanTerminator + AN // ArabicNumber + CS // CommonSeparator + B // ParagraphSeparator + S // SegmentSeparator + WS // WhiteSpace + ON // OtherNeutral + BN // BoundaryNeutral + NSM // NonspacingMark + AL // ArabicLetter + Control // Control LRO - PDI + + numClass + + LRO // LeftToRightOverride + RLO // RightToLeftOverride + LRE // LeftToRightEmbedding + RLE // RightToLeftEmbedding + PDF // PopDirectionalFormat + LRI // LeftToRightIsolate + RLI // RightToLeftIsolate + FSI // FirstStrongIsolate + PDI // PopDirectionalIsolate + + unknownClass = ^Class(0) +) + +var controlToClass = map[rune]Class{ + 0x202D: LRO, // LeftToRightOverride, + 0x202E: RLO, // RightToLeftOverride, + 0x202A: LRE, // LeftToRightEmbedding, + 0x202B: RLE, // RightToLeftEmbedding, + 0x202C: PDF, // PopDirectionalFormat, + 0x2066: LRI, // LeftToRightIsolate, + 0x2067: RLI, // RightToLeftIsolate, + 0x2068: FSI, // FirstStrongIsolate, + 0x2069: PDI, // PopDirectionalIsolate, +} + +// A trie entry has the following bits: +// 7..5 XOR mask for brackets +// 4 1: Bracket open, 0: Bracket close +// 3..0 Class type + +const ( + openMask = 0x10 + xorMaskShift = 5 +) diff --git a/vendor/golang.org/x/text/unicode/bidi/prop.go b/vendor/golang.org/x/text/unicode/bidi/prop.go new file mode 100644 index 00000000000..7c9484e1f50 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/prop.go @@ -0,0 +1,206 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bidi + +import "unicode/utf8" + +// Properties provides access to BiDi properties of runes. +type Properties struct { + entry uint8 + last uint8 +} + +var trie = newBidiTrie(0) + +// TODO: using this for bidirule reduces the running time by about 5%. Consider +// if this is worth exposing or if we can find a way to speed up the Class +// method. +// +// // CompactClass is like Class, but maps all of the BiDi control classes +// // (LRO, RLO, LRE, RLE, PDF, LRI, RLI, FSI, PDI) to the class Control. +// func (p Properties) CompactClass() Class { +// return Class(p.entry & 0x0F) +// } + +// Class returns the Bidi class for p. +func (p Properties) Class() Class { + c := Class(p.entry & 0x0F) + if c == Control { + c = controlByteToClass[p.last&0xF] + } + return c +} + +// IsBracket reports whether the rune is a bracket. +func (p Properties) IsBracket() bool { return p.entry&0xF0 != 0 } + +// IsOpeningBracket reports whether the rune is an opening bracket. +// IsBracket must return true. +func (p Properties) IsOpeningBracket() bool { return p.entry&openMask != 0 } + +// TODO: find a better API and expose. +func (p Properties) reverseBracket(r rune) rune { + return xorMasks[p.entry>>xorMaskShift] ^ r +} + +var controlByteToClass = [16]Class{ + 0xD: LRO, // U+202D LeftToRightOverride, + 0xE: RLO, // U+202E RightToLeftOverride, + 0xA: LRE, // U+202A LeftToRightEmbedding, + 0xB: RLE, // U+202B RightToLeftEmbedding, + 0xC: PDF, // U+202C PopDirectionalFormat, + 0x6: LRI, // U+2066 LeftToRightIsolate, + 0x7: RLI, // U+2067 RightToLeftIsolate, + 0x8: FSI, // U+2068 FirstStrongIsolate, + 0x9: PDI, // U+2069 PopDirectionalIsolate, +} + +// LookupRune returns properties for r. +func LookupRune(r rune) (p Properties, size int) { + var buf [4]byte + n := utf8.EncodeRune(buf[:], r) + return Lookup(buf[:n]) +} + +// TODO: these lookup methods are based on the generated trie code. The returned +// sizes have slightly different semantics from the generated code, in that it +// always returns size==1 for an illegal UTF-8 byte (instead of the length +// of the maximum invalid subsequence). Most Transformers, like unicode/norm, +// leave invalid UTF-8 untouched, in which case it has performance benefits to +// do so (without changing the semantics). Bidi requires the semantics used here +// for the bidirule implementation to be compatible with the Go semantics. +// They ultimately should perhaps be adopted by all trie implementations, for +// convenience sake. +// This unrolled code also boosts performance of the secure/bidirule package by +// about 30%. +// So, to remove this code: +// - add option to trie generator to define return type. +// - always return 1 byte size for ill-formed UTF-8 runes. + +// Lookup returns properties for the first rune in s and the width in bytes of +// its encoding. The size will be 0 if s does not hold enough bytes to complete +// the encoding. +func Lookup(s []byte) (p Properties, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return Properties{entry: bidiValues[c0]}, 1 + case c0 < 0xC2: + return Properties{}, 1 + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return Properties{}, 1 + } + o = uint32(i)<<6 + uint32(c2) + i = bidiIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4 + } + // Illegal rune + return Properties{}, 1 +} + +// LookupString returns properties for the first rune in s and the width in +// bytes of its encoding. The size will be 0 if s does not hold enough bytes to +// complete the encoding. +func LookupString(s string) (p Properties, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return Properties{entry: bidiValues[c0]}, 1 + case c0 < 0xC2: + return Properties{}, 1 + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return Properties{}, 1 + } + o = uint32(i)<<6 + uint32(c2) + i = bidiIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4 + } + // Illegal rune + return Properties{}, 1 +} diff --git a/vendor/golang.org/x/text/unicode/bidi/tables.go b/vendor/golang.org/x/text/unicode/bidi/tables.go new file mode 100644 index 00000000000..2d4dde07c54 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/tables.go @@ -0,0 +1,1779 @@ +// This file was generated by go generate; DO NOT EDIT + +package bidi + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "9.0.0" + +// xorMasks contains masks to be xor-ed with brackets to get the reverse +// version. +var xorMasks = []int32{ // 8 elements + 0, 1, 6, 7, 3, 15, 29, 63, +} // Size: 56 bytes + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *bidiTrie) lookup(s []byte) (v uint8, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return bidiValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = bidiIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *bidiTrie) lookupUnsafe(s []byte) uint8 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return bidiValues[c0] + } + i := bidiIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *bidiTrie) lookupString(s string) (v uint8, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return bidiValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = bidiIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *bidiTrie) lookupStringUnsafe(s string) uint8 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return bidiValues[c0] + } + i := bidiIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// bidiTrie. Total size: 15744 bytes (15.38 KiB). Checksum: b4c3b70954803b86. +type bidiTrie struct{} + +func newBidiTrie(i int) *bidiTrie { + return &bidiTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *bidiTrie) lookupValue(n uint32, b byte) uint8 { + switch { + default: + return uint8(bidiValues[n<<6+uint32(b)]) + } +} + +// bidiValues: 222 blocks, 14208 entries, 14208 bytes +// The third block is the zero block. +var bidiValues = [14208]uint8{ + // Block 0x0, offset 0x0 + 0x00: 0x000b, 0x01: 0x000b, 0x02: 0x000b, 0x03: 0x000b, 0x04: 0x000b, 0x05: 0x000b, + 0x06: 0x000b, 0x07: 0x000b, 0x08: 0x000b, 0x09: 0x0008, 0x0a: 0x0007, 0x0b: 0x0008, + 0x0c: 0x0009, 0x0d: 0x0007, 0x0e: 0x000b, 0x0f: 0x000b, 0x10: 0x000b, 0x11: 0x000b, + 0x12: 0x000b, 0x13: 0x000b, 0x14: 0x000b, 0x15: 0x000b, 0x16: 0x000b, 0x17: 0x000b, + 0x18: 0x000b, 0x19: 0x000b, 0x1a: 0x000b, 0x1b: 0x000b, 0x1c: 0x0007, 0x1d: 0x0007, + 0x1e: 0x0007, 0x1f: 0x0008, 0x20: 0x0009, 0x21: 0x000a, 0x22: 0x000a, 0x23: 0x0004, + 0x24: 0x0004, 0x25: 0x0004, 0x26: 0x000a, 0x27: 0x000a, 0x28: 0x003a, 0x29: 0x002a, + 0x2a: 0x000a, 0x2b: 0x0003, 0x2c: 0x0006, 0x2d: 0x0003, 0x2e: 0x0006, 0x2f: 0x0006, + 0x30: 0x0002, 0x31: 0x0002, 0x32: 0x0002, 0x33: 0x0002, 0x34: 0x0002, 0x35: 0x0002, + 0x36: 0x0002, 0x37: 0x0002, 0x38: 0x0002, 0x39: 0x0002, 0x3a: 0x0006, 0x3b: 0x000a, + 0x3c: 0x000a, 0x3d: 0x000a, 0x3e: 0x000a, 0x3f: 0x000a, + // Block 0x1, offset 0x40 + 0x40: 0x000a, + 0x5b: 0x005a, 0x5c: 0x000a, 0x5d: 0x004a, + 0x5e: 0x000a, 0x5f: 0x000a, 0x60: 0x000a, + 0x7b: 0x005a, + 0x7c: 0x000a, 0x7d: 0x004a, 0x7e: 0x000a, 0x7f: 0x000b, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc0: 0x000b, 0xc1: 0x000b, 0xc2: 0x000b, 0xc3: 0x000b, 0xc4: 0x000b, 0xc5: 0x0007, + 0xc6: 0x000b, 0xc7: 0x000b, 0xc8: 0x000b, 0xc9: 0x000b, 0xca: 0x000b, 0xcb: 0x000b, + 0xcc: 0x000b, 0xcd: 0x000b, 0xce: 0x000b, 0xcf: 0x000b, 0xd0: 0x000b, 0xd1: 0x000b, + 0xd2: 0x000b, 0xd3: 0x000b, 0xd4: 0x000b, 0xd5: 0x000b, 0xd6: 0x000b, 0xd7: 0x000b, + 0xd8: 0x000b, 0xd9: 0x000b, 0xda: 0x000b, 0xdb: 0x000b, 0xdc: 0x000b, 0xdd: 0x000b, + 0xde: 0x000b, 0xdf: 0x000b, 0xe0: 0x0006, 0xe1: 0x000a, 0xe2: 0x0004, 0xe3: 0x0004, + 0xe4: 0x0004, 0xe5: 0x0004, 0xe6: 0x000a, 0xe7: 0x000a, 0xe8: 0x000a, 0xe9: 0x000a, + 0xeb: 0x000a, 0xec: 0x000a, 0xed: 0x000b, 0xee: 0x000a, 0xef: 0x000a, + 0xf0: 0x0004, 0xf1: 0x0004, 0xf2: 0x0002, 0xf3: 0x0002, 0xf4: 0x000a, + 0xf6: 0x000a, 0xf7: 0x000a, 0xf8: 0x000a, 0xf9: 0x0002, 0xfb: 0x000a, + 0xfc: 0x000a, 0xfd: 0x000a, 0xfe: 0x000a, 0xff: 0x000a, + // Block 0x4, offset 0x100 + 0x117: 0x000a, + 0x137: 0x000a, + // Block 0x5, offset 0x140 + 0x179: 0x000a, 0x17a: 0x000a, + // Block 0x6, offset 0x180 + 0x182: 0x000a, 0x183: 0x000a, 0x184: 0x000a, 0x185: 0x000a, + 0x186: 0x000a, 0x187: 0x000a, 0x188: 0x000a, 0x189: 0x000a, 0x18a: 0x000a, 0x18b: 0x000a, + 0x18c: 0x000a, 0x18d: 0x000a, 0x18e: 0x000a, 0x18f: 0x000a, + 0x192: 0x000a, 0x193: 0x000a, 0x194: 0x000a, 0x195: 0x000a, 0x196: 0x000a, 0x197: 0x000a, + 0x198: 0x000a, 0x199: 0x000a, 0x19a: 0x000a, 0x19b: 0x000a, 0x19c: 0x000a, 0x19d: 0x000a, + 0x19e: 0x000a, 0x19f: 0x000a, + 0x1a5: 0x000a, 0x1a6: 0x000a, 0x1a7: 0x000a, 0x1a8: 0x000a, 0x1a9: 0x000a, + 0x1aa: 0x000a, 0x1ab: 0x000a, 0x1ac: 0x000a, 0x1ad: 0x000a, 0x1af: 0x000a, + 0x1b0: 0x000a, 0x1b1: 0x000a, 0x1b2: 0x000a, 0x1b3: 0x000a, 0x1b4: 0x000a, 0x1b5: 0x000a, + 0x1b6: 0x000a, 0x1b7: 0x000a, 0x1b8: 0x000a, 0x1b9: 0x000a, 0x1ba: 0x000a, 0x1bb: 0x000a, + 0x1bc: 0x000a, 0x1bd: 0x000a, 0x1be: 0x000a, 0x1bf: 0x000a, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x000c, 0x1c1: 0x000c, 0x1c2: 0x000c, 0x1c3: 0x000c, 0x1c4: 0x000c, 0x1c5: 0x000c, + 0x1c6: 0x000c, 0x1c7: 0x000c, 0x1c8: 0x000c, 0x1c9: 0x000c, 0x1ca: 0x000c, 0x1cb: 0x000c, + 0x1cc: 0x000c, 0x1cd: 0x000c, 0x1ce: 0x000c, 0x1cf: 0x000c, 0x1d0: 0x000c, 0x1d1: 0x000c, + 0x1d2: 0x000c, 0x1d3: 0x000c, 0x1d4: 0x000c, 0x1d5: 0x000c, 0x1d6: 0x000c, 0x1d7: 0x000c, + 0x1d8: 0x000c, 0x1d9: 0x000c, 0x1da: 0x000c, 0x1db: 0x000c, 0x1dc: 0x000c, 0x1dd: 0x000c, + 0x1de: 0x000c, 0x1df: 0x000c, 0x1e0: 0x000c, 0x1e1: 0x000c, 0x1e2: 0x000c, 0x1e3: 0x000c, + 0x1e4: 0x000c, 0x1e5: 0x000c, 0x1e6: 0x000c, 0x1e7: 0x000c, 0x1e8: 0x000c, 0x1e9: 0x000c, + 0x1ea: 0x000c, 0x1eb: 0x000c, 0x1ec: 0x000c, 0x1ed: 0x000c, 0x1ee: 0x000c, 0x1ef: 0x000c, + 0x1f0: 0x000c, 0x1f1: 0x000c, 0x1f2: 0x000c, 0x1f3: 0x000c, 0x1f4: 0x000c, 0x1f5: 0x000c, + 0x1f6: 0x000c, 0x1f7: 0x000c, 0x1f8: 0x000c, 0x1f9: 0x000c, 0x1fa: 0x000c, 0x1fb: 0x000c, + 0x1fc: 0x000c, 0x1fd: 0x000c, 0x1fe: 0x000c, 0x1ff: 0x000c, + // Block 0x8, offset 0x200 + 0x200: 0x000c, 0x201: 0x000c, 0x202: 0x000c, 0x203: 0x000c, 0x204: 0x000c, 0x205: 0x000c, + 0x206: 0x000c, 0x207: 0x000c, 0x208: 0x000c, 0x209: 0x000c, 0x20a: 0x000c, 0x20b: 0x000c, + 0x20c: 0x000c, 0x20d: 0x000c, 0x20e: 0x000c, 0x20f: 0x000c, 0x210: 0x000c, 0x211: 0x000c, + 0x212: 0x000c, 0x213: 0x000c, 0x214: 0x000c, 0x215: 0x000c, 0x216: 0x000c, 0x217: 0x000c, + 0x218: 0x000c, 0x219: 0x000c, 0x21a: 0x000c, 0x21b: 0x000c, 0x21c: 0x000c, 0x21d: 0x000c, + 0x21e: 0x000c, 0x21f: 0x000c, 0x220: 0x000c, 0x221: 0x000c, 0x222: 0x000c, 0x223: 0x000c, + 0x224: 0x000c, 0x225: 0x000c, 0x226: 0x000c, 0x227: 0x000c, 0x228: 0x000c, 0x229: 0x000c, + 0x22a: 0x000c, 0x22b: 0x000c, 0x22c: 0x000c, 0x22d: 0x000c, 0x22e: 0x000c, 0x22f: 0x000c, + 0x234: 0x000a, 0x235: 0x000a, + 0x23e: 0x000a, + // Block 0x9, offset 0x240 + 0x244: 0x000a, 0x245: 0x000a, + 0x247: 0x000a, + // Block 0xa, offset 0x280 + 0x2b6: 0x000a, + // Block 0xb, offset 0x2c0 + 0x2c3: 0x000c, 0x2c4: 0x000c, 0x2c5: 0x000c, + 0x2c6: 0x000c, 0x2c7: 0x000c, 0x2c8: 0x000c, 0x2c9: 0x000c, + // Block 0xc, offset 0x300 + 0x30a: 0x000a, + 0x30d: 0x000a, 0x30e: 0x000a, 0x30f: 0x0004, 0x310: 0x0001, 0x311: 0x000c, + 0x312: 0x000c, 0x313: 0x000c, 0x314: 0x000c, 0x315: 0x000c, 0x316: 0x000c, 0x317: 0x000c, + 0x318: 0x000c, 0x319: 0x000c, 0x31a: 0x000c, 0x31b: 0x000c, 0x31c: 0x000c, 0x31d: 0x000c, + 0x31e: 0x000c, 0x31f: 0x000c, 0x320: 0x000c, 0x321: 0x000c, 0x322: 0x000c, 0x323: 0x000c, + 0x324: 0x000c, 0x325: 0x000c, 0x326: 0x000c, 0x327: 0x000c, 0x328: 0x000c, 0x329: 0x000c, + 0x32a: 0x000c, 0x32b: 0x000c, 0x32c: 0x000c, 0x32d: 0x000c, 0x32e: 0x000c, 0x32f: 0x000c, + 0x330: 0x000c, 0x331: 0x000c, 0x332: 0x000c, 0x333: 0x000c, 0x334: 0x000c, 0x335: 0x000c, + 0x336: 0x000c, 0x337: 0x000c, 0x338: 0x000c, 0x339: 0x000c, 0x33a: 0x000c, 0x33b: 0x000c, + 0x33c: 0x000c, 0x33d: 0x000c, 0x33e: 0x0001, 0x33f: 0x000c, + // Block 0xd, offset 0x340 + 0x340: 0x0001, 0x341: 0x000c, 0x342: 0x000c, 0x343: 0x0001, 0x344: 0x000c, 0x345: 0x000c, + 0x346: 0x0001, 0x347: 0x000c, 0x348: 0x0001, 0x349: 0x0001, 0x34a: 0x0001, 0x34b: 0x0001, + 0x34c: 0x0001, 0x34d: 0x0001, 0x34e: 0x0001, 0x34f: 0x0001, 0x350: 0x0001, 0x351: 0x0001, + 0x352: 0x0001, 0x353: 0x0001, 0x354: 0x0001, 0x355: 0x0001, 0x356: 0x0001, 0x357: 0x0001, + 0x358: 0x0001, 0x359: 0x0001, 0x35a: 0x0001, 0x35b: 0x0001, 0x35c: 0x0001, 0x35d: 0x0001, + 0x35e: 0x0001, 0x35f: 0x0001, 0x360: 0x0001, 0x361: 0x0001, 0x362: 0x0001, 0x363: 0x0001, + 0x364: 0x0001, 0x365: 0x0001, 0x366: 0x0001, 0x367: 0x0001, 0x368: 0x0001, 0x369: 0x0001, + 0x36a: 0x0001, 0x36b: 0x0001, 0x36c: 0x0001, 0x36d: 0x0001, 0x36e: 0x0001, 0x36f: 0x0001, + 0x370: 0x0001, 0x371: 0x0001, 0x372: 0x0001, 0x373: 0x0001, 0x374: 0x0001, 0x375: 0x0001, + 0x376: 0x0001, 0x377: 0x0001, 0x378: 0x0001, 0x379: 0x0001, 0x37a: 0x0001, 0x37b: 0x0001, + 0x37c: 0x0001, 0x37d: 0x0001, 0x37e: 0x0001, 0x37f: 0x0001, + // Block 0xe, offset 0x380 + 0x380: 0x0005, 0x381: 0x0005, 0x382: 0x0005, 0x383: 0x0005, 0x384: 0x0005, 0x385: 0x0005, + 0x386: 0x000a, 0x387: 0x000a, 0x388: 0x000d, 0x389: 0x0004, 0x38a: 0x0004, 0x38b: 0x000d, + 0x38c: 0x0006, 0x38d: 0x000d, 0x38e: 0x000a, 0x38f: 0x000a, 0x390: 0x000c, 0x391: 0x000c, + 0x392: 0x000c, 0x393: 0x000c, 0x394: 0x000c, 0x395: 0x000c, 0x396: 0x000c, 0x397: 0x000c, + 0x398: 0x000c, 0x399: 0x000c, 0x39a: 0x000c, 0x39b: 0x000d, 0x39c: 0x000d, 0x39d: 0x000d, + 0x39e: 0x000d, 0x39f: 0x000d, 0x3a0: 0x000d, 0x3a1: 0x000d, 0x3a2: 0x000d, 0x3a3: 0x000d, + 0x3a4: 0x000d, 0x3a5: 0x000d, 0x3a6: 0x000d, 0x3a7: 0x000d, 0x3a8: 0x000d, 0x3a9: 0x000d, + 0x3aa: 0x000d, 0x3ab: 0x000d, 0x3ac: 0x000d, 0x3ad: 0x000d, 0x3ae: 0x000d, 0x3af: 0x000d, + 0x3b0: 0x000d, 0x3b1: 0x000d, 0x3b2: 0x000d, 0x3b3: 0x000d, 0x3b4: 0x000d, 0x3b5: 0x000d, + 0x3b6: 0x000d, 0x3b7: 0x000d, 0x3b8: 0x000d, 0x3b9: 0x000d, 0x3ba: 0x000d, 0x3bb: 0x000d, + 0x3bc: 0x000d, 0x3bd: 0x000d, 0x3be: 0x000d, 0x3bf: 0x000d, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x000d, 0x3c1: 0x000d, 0x3c2: 0x000d, 0x3c3: 0x000d, 0x3c4: 0x000d, 0x3c5: 0x000d, + 0x3c6: 0x000d, 0x3c7: 0x000d, 0x3c8: 0x000d, 0x3c9: 0x000d, 0x3ca: 0x000d, 0x3cb: 0x000c, + 0x3cc: 0x000c, 0x3cd: 0x000c, 0x3ce: 0x000c, 0x3cf: 0x000c, 0x3d0: 0x000c, 0x3d1: 0x000c, + 0x3d2: 0x000c, 0x3d3: 0x000c, 0x3d4: 0x000c, 0x3d5: 0x000c, 0x3d6: 0x000c, 0x3d7: 0x000c, + 0x3d8: 0x000c, 0x3d9: 0x000c, 0x3da: 0x000c, 0x3db: 0x000c, 0x3dc: 0x000c, 0x3dd: 0x000c, + 0x3de: 0x000c, 0x3df: 0x000c, 0x3e0: 0x0005, 0x3e1: 0x0005, 0x3e2: 0x0005, 0x3e3: 0x0005, + 0x3e4: 0x0005, 0x3e5: 0x0005, 0x3e6: 0x0005, 0x3e7: 0x0005, 0x3e8: 0x0005, 0x3e9: 0x0005, + 0x3ea: 0x0004, 0x3eb: 0x0005, 0x3ec: 0x0005, 0x3ed: 0x000d, 0x3ee: 0x000d, 0x3ef: 0x000d, + 0x3f0: 0x000c, 0x3f1: 0x000d, 0x3f2: 0x000d, 0x3f3: 0x000d, 0x3f4: 0x000d, 0x3f5: 0x000d, + 0x3f6: 0x000d, 0x3f7: 0x000d, 0x3f8: 0x000d, 0x3f9: 0x000d, 0x3fa: 0x000d, 0x3fb: 0x000d, + 0x3fc: 0x000d, 0x3fd: 0x000d, 0x3fe: 0x000d, 0x3ff: 0x000d, + // Block 0x10, offset 0x400 + 0x400: 0x000d, 0x401: 0x000d, 0x402: 0x000d, 0x403: 0x000d, 0x404: 0x000d, 0x405: 0x000d, + 0x406: 0x000d, 0x407: 0x000d, 0x408: 0x000d, 0x409: 0x000d, 0x40a: 0x000d, 0x40b: 0x000d, + 0x40c: 0x000d, 0x40d: 0x000d, 0x40e: 0x000d, 0x40f: 0x000d, 0x410: 0x000d, 0x411: 0x000d, + 0x412: 0x000d, 0x413: 0x000d, 0x414: 0x000d, 0x415: 0x000d, 0x416: 0x000d, 0x417: 0x000d, + 0x418: 0x000d, 0x419: 0x000d, 0x41a: 0x000d, 0x41b: 0x000d, 0x41c: 0x000d, 0x41d: 0x000d, + 0x41e: 0x000d, 0x41f: 0x000d, 0x420: 0x000d, 0x421: 0x000d, 0x422: 0x000d, 0x423: 0x000d, + 0x424: 0x000d, 0x425: 0x000d, 0x426: 0x000d, 0x427: 0x000d, 0x428: 0x000d, 0x429: 0x000d, + 0x42a: 0x000d, 0x42b: 0x000d, 0x42c: 0x000d, 0x42d: 0x000d, 0x42e: 0x000d, 0x42f: 0x000d, + 0x430: 0x000d, 0x431: 0x000d, 0x432: 0x000d, 0x433: 0x000d, 0x434: 0x000d, 0x435: 0x000d, + 0x436: 0x000d, 0x437: 0x000d, 0x438: 0x000d, 0x439: 0x000d, 0x43a: 0x000d, 0x43b: 0x000d, + 0x43c: 0x000d, 0x43d: 0x000d, 0x43e: 0x000d, 0x43f: 0x000d, + // Block 0x11, offset 0x440 + 0x440: 0x000d, 0x441: 0x000d, 0x442: 0x000d, 0x443: 0x000d, 0x444: 0x000d, 0x445: 0x000d, + 0x446: 0x000d, 0x447: 0x000d, 0x448: 0x000d, 0x449: 0x000d, 0x44a: 0x000d, 0x44b: 0x000d, + 0x44c: 0x000d, 0x44d: 0x000d, 0x44e: 0x000d, 0x44f: 0x000d, 0x450: 0x000d, 0x451: 0x000d, + 0x452: 0x000d, 0x453: 0x000d, 0x454: 0x000d, 0x455: 0x000d, 0x456: 0x000c, 0x457: 0x000c, + 0x458: 0x000c, 0x459: 0x000c, 0x45a: 0x000c, 0x45b: 0x000c, 0x45c: 0x000c, 0x45d: 0x0005, + 0x45e: 0x000a, 0x45f: 0x000c, 0x460: 0x000c, 0x461: 0x000c, 0x462: 0x000c, 0x463: 0x000c, + 0x464: 0x000c, 0x465: 0x000d, 0x466: 0x000d, 0x467: 0x000c, 0x468: 0x000c, 0x469: 0x000a, + 0x46a: 0x000c, 0x46b: 0x000c, 0x46c: 0x000c, 0x46d: 0x000c, 0x46e: 0x000d, 0x46f: 0x000d, + 0x470: 0x0002, 0x471: 0x0002, 0x472: 0x0002, 0x473: 0x0002, 0x474: 0x0002, 0x475: 0x0002, + 0x476: 0x0002, 0x477: 0x0002, 0x478: 0x0002, 0x479: 0x0002, 0x47a: 0x000d, 0x47b: 0x000d, + 0x47c: 0x000d, 0x47d: 0x000d, 0x47e: 0x000d, 0x47f: 0x000d, + // Block 0x12, offset 0x480 + 0x480: 0x000d, 0x481: 0x000d, 0x482: 0x000d, 0x483: 0x000d, 0x484: 0x000d, 0x485: 0x000d, + 0x486: 0x000d, 0x487: 0x000d, 0x488: 0x000d, 0x489: 0x000d, 0x48a: 0x000d, 0x48b: 0x000d, + 0x48c: 0x000d, 0x48d: 0x000d, 0x48e: 0x000d, 0x48f: 0x000d, 0x490: 0x000d, 0x491: 0x000c, + 0x492: 0x000d, 0x493: 0x000d, 0x494: 0x000d, 0x495: 0x000d, 0x496: 0x000d, 0x497: 0x000d, + 0x498: 0x000d, 0x499: 0x000d, 0x49a: 0x000d, 0x49b: 0x000d, 0x49c: 0x000d, 0x49d: 0x000d, + 0x49e: 0x000d, 0x49f: 0x000d, 0x4a0: 0x000d, 0x4a1: 0x000d, 0x4a2: 0x000d, 0x4a3: 0x000d, + 0x4a4: 0x000d, 0x4a5: 0x000d, 0x4a6: 0x000d, 0x4a7: 0x000d, 0x4a8: 0x000d, 0x4a9: 0x000d, + 0x4aa: 0x000d, 0x4ab: 0x000d, 0x4ac: 0x000d, 0x4ad: 0x000d, 0x4ae: 0x000d, 0x4af: 0x000d, + 0x4b0: 0x000c, 0x4b1: 0x000c, 0x4b2: 0x000c, 0x4b3: 0x000c, 0x4b4: 0x000c, 0x4b5: 0x000c, + 0x4b6: 0x000c, 0x4b7: 0x000c, 0x4b8: 0x000c, 0x4b9: 0x000c, 0x4ba: 0x000c, 0x4bb: 0x000c, + 0x4bc: 0x000c, 0x4bd: 0x000c, 0x4be: 0x000c, 0x4bf: 0x000c, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x000c, 0x4c1: 0x000c, 0x4c2: 0x000c, 0x4c3: 0x000c, 0x4c4: 0x000c, 0x4c5: 0x000c, + 0x4c6: 0x000c, 0x4c7: 0x000c, 0x4c8: 0x000c, 0x4c9: 0x000c, 0x4ca: 0x000c, 0x4cb: 0x000d, + 0x4cc: 0x000d, 0x4cd: 0x000d, 0x4ce: 0x000d, 0x4cf: 0x000d, 0x4d0: 0x000d, 0x4d1: 0x000d, + 0x4d2: 0x000d, 0x4d3: 0x000d, 0x4d4: 0x000d, 0x4d5: 0x000d, 0x4d6: 0x000d, 0x4d7: 0x000d, + 0x4d8: 0x000d, 0x4d9: 0x000d, 0x4da: 0x000d, 0x4db: 0x000d, 0x4dc: 0x000d, 0x4dd: 0x000d, + 0x4de: 0x000d, 0x4df: 0x000d, 0x4e0: 0x000d, 0x4e1: 0x000d, 0x4e2: 0x000d, 0x4e3: 0x000d, + 0x4e4: 0x000d, 0x4e5: 0x000d, 0x4e6: 0x000d, 0x4e7: 0x000d, 0x4e8: 0x000d, 0x4e9: 0x000d, + 0x4ea: 0x000d, 0x4eb: 0x000d, 0x4ec: 0x000d, 0x4ed: 0x000d, 0x4ee: 0x000d, 0x4ef: 0x000d, + 0x4f0: 0x000d, 0x4f1: 0x000d, 0x4f2: 0x000d, 0x4f3: 0x000d, 0x4f4: 0x000d, 0x4f5: 0x000d, + 0x4f6: 0x000d, 0x4f7: 0x000d, 0x4f8: 0x000d, 0x4f9: 0x000d, 0x4fa: 0x000d, 0x4fb: 0x000d, + 0x4fc: 0x000d, 0x4fd: 0x000d, 0x4fe: 0x000d, 0x4ff: 0x000d, + // Block 0x14, offset 0x500 + 0x500: 0x000d, 0x501: 0x000d, 0x502: 0x000d, 0x503: 0x000d, 0x504: 0x000d, 0x505: 0x000d, + 0x506: 0x000d, 0x507: 0x000d, 0x508: 0x000d, 0x509: 0x000d, 0x50a: 0x000d, 0x50b: 0x000d, + 0x50c: 0x000d, 0x50d: 0x000d, 0x50e: 0x000d, 0x50f: 0x000d, 0x510: 0x000d, 0x511: 0x000d, + 0x512: 0x000d, 0x513: 0x000d, 0x514: 0x000d, 0x515: 0x000d, 0x516: 0x000d, 0x517: 0x000d, + 0x518: 0x000d, 0x519: 0x000d, 0x51a: 0x000d, 0x51b: 0x000d, 0x51c: 0x000d, 0x51d: 0x000d, + 0x51e: 0x000d, 0x51f: 0x000d, 0x520: 0x000d, 0x521: 0x000d, 0x522: 0x000d, 0x523: 0x000d, + 0x524: 0x000d, 0x525: 0x000d, 0x526: 0x000c, 0x527: 0x000c, 0x528: 0x000c, 0x529: 0x000c, + 0x52a: 0x000c, 0x52b: 0x000c, 0x52c: 0x000c, 0x52d: 0x000c, 0x52e: 0x000c, 0x52f: 0x000c, + 0x530: 0x000c, 0x531: 0x000d, 0x532: 0x000d, 0x533: 0x000d, 0x534: 0x000d, 0x535: 0x000d, + 0x536: 0x000d, 0x537: 0x000d, 0x538: 0x000d, 0x539: 0x000d, 0x53a: 0x000d, 0x53b: 0x000d, + 0x53c: 0x000d, 0x53d: 0x000d, 0x53e: 0x000d, 0x53f: 0x000d, + // Block 0x15, offset 0x540 + 0x540: 0x0001, 0x541: 0x0001, 0x542: 0x0001, 0x543: 0x0001, 0x544: 0x0001, 0x545: 0x0001, + 0x546: 0x0001, 0x547: 0x0001, 0x548: 0x0001, 0x549: 0x0001, 0x54a: 0x0001, 0x54b: 0x0001, + 0x54c: 0x0001, 0x54d: 0x0001, 0x54e: 0x0001, 0x54f: 0x0001, 0x550: 0x0001, 0x551: 0x0001, + 0x552: 0x0001, 0x553: 0x0001, 0x554: 0x0001, 0x555: 0x0001, 0x556: 0x0001, 0x557: 0x0001, + 0x558: 0x0001, 0x559: 0x0001, 0x55a: 0x0001, 0x55b: 0x0001, 0x55c: 0x0001, 0x55d: 0x0001, + 0x55e: 0x0001, 0x55f: 0x0001, 0x560: 0x0001, 0x561: 0x0001, 0x562: 0x0001, 0x563: 0x0001, + 0x564: 0x0001, 0x565: 0x0001, 0x566: 0x0001, 0x567: 0x0001, 0x568: 0x0001, 0x569: 0x0001, + 0x56a: 0x0001, 0x56b: 0x000c, 0x56c: 0x000c, 0x56d: 0x000c, 0x56e: 0x000c, 0x56f: 0x000c, + 0x570: 0x000c, 0x571: 0x000c, 0x572: 0x000c, 0x573: 0x000c, 0x574: 0x0001, 0x575: 0x0001, + 0x576: 0x000a, 0x577: 0x000a, 0x578: 0x000a, 0x579: 0x000a, 0x57a: 0x0001, 0x57b: 0x0001, + 0x57c: 0x0001, 0x57d: 0x0001, 0x57e: 0x0001, 0x57f: 0x0001, + // Block 0x16, offset 0x580 + 0x580: 0x0001, 0x581: 0x0001, 0x582: 0x0001, 0x583: 0x0001, 0x584: 0x0001, 0x585: 0x0001, + 0x586: 0x0001, 0x587: 0x0001, 0x588: 0x0001, 0x589: 0x0001, 0x58a: 0x0001, 0x58b: 0x0001, + 0x58c: 0x0001, 0x58d: 0x0001, 0x58e: 0x0001, 0x58f: 0x0001, 0x590: 0x0001, 0x591: 0x0001, + 0x592: 0x0001, 0x593: 0x0001, 0x594: 0x0001, 0x595: 0x0001, 0x596: 0x000c, 0x597: 0x000c, + 0x598: 0x000c, 0x599: 0x000c, 0x59a: 0x0001, 0x59b: 0x000c, 0x59c: 0x000c, 0x59d: 0x000c, + 0x59e: 0x000c, 0x59f: 0x000c, 0x5a0: 0x000c, 0x5a1: 0x000c, 0x5a2: 0x000c, 0x5a3: 0x000c, + 0x5a4: 0x0001, 0x5a5: 0x000c, 0x5a6: 0x000c, 0x5a7: 0x000c, 0x5a8: 0x0001, 0x5a9: 0x000c, + 0x5aa: 0x000c, 0x5ab: 0x000c, 0x5ac: 0x000c, 0x5ad: 0x000c, 0x5ae: 0x0001, 0x5af: 0x0001, + 0x5b0: 0x0001, 0x5b1: 0x0001, 0x5b2: 0x0001, 0x5b3: 0x0001, 0x5b4: 0x0001, 0x5b5: 0x0001, + 0x5b6: 0x0001, 0x5b7: 0x0001, 0x5b8: 0x0001, 0x5b9: 0x0001, 0x5ba: 0x0001, 0x5bb: 0x0001, + 0x5bc: 0x0001, 0x5bd: 0x0001, 0x5be: 0x0001, 0x5bf: 0x0001, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x0001, 0x5c1: 0x0001, 0x5c2: 0x0001, 0x5c3: 0x0001, 0x5c4: 0x0001, 0x5c5: 0x0001, + 0x5c6: 0x0001, 0x5c7: 0x0001, 0x5c8: 0x0001, 0x5c9: 0x0001, 0x5ca: 0x0001, 0x5cb: 0x0001, + 0x5cc: 0x0001, 0x5cd: 0x0001, 0x5ce: 0x0001, 0x5cf: 0x0001, 0x5d0: 0x0001, 0x5d1: 0x0001, + 0x5d2: 0x0001, 0x5d3: 0x0001, 0x5d4: 0x0001, 0x5d5: 0x0001, 0x5d6: 0x0001, 0x5d7: 0x0001, + 0x5d8: 0x0001, 0x5d9: 0x000c, 0x5da: 0x000c, 0x5db: 0x000c, 0x5dc: 0x0001, 0x5dd: 0x0001, + 0x5de: 0x0001, 0x5df: 0x0001, 0x5e0: 0x0001, 0x5e1: 0x0001, 0x5e2: 0x0001, 0x5e3: 0x0001, + 0x5e4: 0x0001, 0x5e5: 0x0001, 0x5e6: 0x0001, 0x5e7: 0x0001, 0x5e8: 0x0001, 0x5e9: 0x0001, + 0x5ea: 0x0001, 0x5eb: 0x0001, 0x5ec: 0x0001, 0x5ed: 0x0001, 0x5ee: 0x0001, 0x5ef: 0x0001, + 0x5f0: 0x0001, 0x5f1: 0x0001, 0x5f2: 0x0001, 0x5f3: 0x0001, 0x5f4: 0x0001, 0x5f5: 0x0001, + 0x5f6: 0x0001, 0x5f7: 0x0001, 0x5f8: 0x0001, 0x5f9: 0x0001, 0x5fa: 0x0001, 0x5fb: 0x0001, + 0x5fc: 0x0001, 0x5fd: 0x0001, 0x5fe: 0x0001, 0x5ff: 0x0001, + // Block 0x18, offset 0x600 + 0x600: 0x0001, 0x601: 0x0001, 0x602: 0x0001, 0x603: 0x0001, 0x604: 0x0001, 0x605: 0x0001, + 0x606: 0x0001, 0x607: 0x0001, 0x608: 0x0001, 0x609: 0x0001, 0x60a: 0x0001, 0x60b: 0x0001, + 0x60c: 0x0001, 0x60d: 0x0001, 0x60e: 0x0001, 0x60f: 0x0001, 0x610: 0x0001, 0x611: 0x0001, + 0x612: 0x0001, 0x613: 0x0001, 0x614: 0x0001, 0x615: 0x0001, 0x616: 0x0001, 0x617: 0x0001, + 0x618: 0x0001, 0x619: 0x0001, 0x61a: 0x0001, 0x61b: 0x0001, 0x61c: 0x0001, 0x61d: 0x0001, + 0x61e: 0x0001, 0x61f: 0x0001, 0x620: 0x000d, 0x621: 0x000d, 0x622: 0x000d, 0x623: 0x000d, + 0x624: 0x000d, 0x625: 0x000d, 0x626: 0x000d, 0x627: 0x000d, 0x628: 0x000d, 0x629: 0x000d, + 0x62a: 0x000d, 0x62b: 0x000d, 0x62c: 0x000d, 0x62d: 0x000d, 0x62e: 0x000d, 0x62f: 0x000d, + 0x630: 0x000d, 0x631: 0x000d, 0x632: 0x000d, 0x633: 0x000d, 0x634: 0x000d, 0x635: 0x000d, + 0x636: 0x000d, 0x637: 0x000d, 0x638: 0x000d, 0x639: 0x000d, 0x63a: 0x000d, 0x63b: 0x000d, + 0x63c: 0x000d, 0x63d: 0x000d, 0x63e: 0x000d, 0x63f: 0x000d, + // Block 0x19, offset 0x640 + 0x640: 0x000d, 0x641: 0x000d, 0x642: 0x000d, 0x643: 0x000d, 0x644: 0x000d, 0x645: 0x000d, + 0x646: 0x000d, 0x647: 0x000d, 0x648: 0x000d, 0x649: 0x000d, 0x64a: 0x000d, 0x64b: 0x000d, + 0x64c: 0x000d, 0x64d: 0x000d, 0x64e: 0x000d, 0x64f: 0x000d, 0x650: 0x000d, 0x651: 0x000d, + 0x652: 0x000d, 0x653: 0x000d, 0x654: 0x000c, 0x655: 0x000c, 0x656: 0x000c, 0x657: 0x000c, + 0x658: 0x000c, 0x659: 0x000c, 0x65a: 0x000c, 0x65b: 0x000c, 0x65c: 0x000c, 0x65d: 0x000c, + 0x65e: 0x000c, 0x65f: 0x000c, 0x660: 0x000c, 0x661: 0x000c, 0x662: 0x0005, 0x663: 0x000c, + 0x664: 0x000c, 0x665: 0x000c, 0x666: 0x000c, 0x667: 0x000c, 0x668: 0x000c, 0x669: 0x000c, + 0x66a: 0x000c, 0x66b: 0x000c, 0x66c: 0x000c, 0x66d: 0x000c, 0x66e: 0x000c, 0x66f: 0x000c, + 0x670: 0x000c, 0x671: 0x000c, 0x672: 0x000c, 0x673: 0x000c, 0x674: 0x000c, 0x675: 0x000c, + 0x676: 0x000c, 0x677: 0x000c, 0x678: 0x000c, 0x679: 0x000c, 0x67a: 0x000c, 0x67b: 0x000c, + 0x67c: 0x000c, 0x67d: 0x000c, 0x67e: 0x000c, 0x67f: 0x000c, + // Block 0x1a, offset 0x680 + 0x680: 0x000c, 0x681: 0x000c, 0x682: 0x000c, + 0x6ba: 0x000c, + 0x6bc: 0x000c, + // Block 0x1b, offset 0x6c0 + 0x6c1: 0x000c, 0x6c2: 0x000c, 0x6c3: 0x000c, 0x6c4: 0x000c, 0x6c5: 0x000c, + 0x6c6: 0x000c, 0x6c7: 0x000c, 0x6c8: 0x000c, + 0x6cd: 0x000c, 0x6d1: 0x000c, + 0x6d2: 0x000c, 0x6d3: 0x000c, 0x6d4: 0x000c, 0x6d5: 0x000c, 0x6d6: 0x000c, 0x6d7: 0x000c, + 0x6e2: 0x000c, 0x6e3: 0x000c, + // Block 0x1c, offset 0x700 + 0x701: 0x000c, + 0x73c: 0x000c, + // Block 0x1d, offset 0x740 + 0x741: 0x000c, 0x742: 0x000c, 0x743: 0x000c, 0x744: 0x000c, + 0x74d: 0x000c, + 0x762: 0x000c, 0x763: 0x000c, + 0x772: 0x0004, 0x773: 0x0004, + 0x77b: 0x0004, + // Block 0x1e, offset 0x780 + 0x781: 0x000c, 0x782: 0x000c, + 0x7bc: 0x000c, + // Block 0x1f, offset 0x7c0 + 0x7c1: 0x000c, 0x7c2: 0x000c, + 0x7c7: 0x000c, 0x7c8: 0x000c, 0x7cb: 0x000c, + 0x7cc: 0x000c, 0x7cd: 0x000c, 0x7d1: 0x000c, + 0x7f0: 0x000c, 0x7f1: 0x000c, 0x7f5: 0x000c, + // Block 0x20, offset 0x800 + 0x801: 0x000c, 0x802: 0x000c, 0x803: 0x000c, 0x804: 0x000c, 0x805: 0x000c, + 0x807: 0x000c, 0x808: 0x000c, + 0x80d: 0x000c, + 0x822: 0x000c, 0x823: 0x000c, + 0x831: 0x0004, + // Block 0x21, offset 0x840 + 0x841: 0x000c, + 0x87c: 0x000c, 0x87f: 0x000c, + // Block 0x22, offset 0x880 + 0x881: 0x000c, 0x882: 0x000c, 0x883: 0x000c, 0x884: 0x000c, + 0x88d: 0x000c, + 0x896: 0x000c, + 0x8a2: 0x000c, 0x8a3: 0x000c, + // Block 0x23, offset 0x8c0 + 0x8c2: 0x000c, + // Block 0x24, offset 0x900 + 0x900: 0x000c, + 0x90d: 0x000c, + 0x933: 0x000a, 0x934: 0x000a, 0x935: 0x000a, + 0x936: 0x000a, 0x937: 0x000a, 0x938: 0x000a, 0x939: 0x0004, 0x93a: 0x000a, + // Block 0x25, offset 0x940 + 0x940: 0x000c, + 0x97e: 0x000c, 0x97f: 0x000c, + // Block 0x26, offset 0x980 + 0x980: 0x000c, + 0x986: 0x000c, 0x987: 0x000c, 0x988: 0x000c, 0x98a: 0x000c, 0x98b: 0x000c, + 0x98c: 0x000c, 0x98d: 0x000c, + 0x995: 0x000c, 0x996: 0x000c, + 0x9a2: 0x000c, 0x9a3: 0x000c, + 0x9b8: 0x000a, 0x9b9: 0x000a, 0x9ba: 0x000a, 0x9bb: 0x000a, + 0x9bc: 0x000a, 0x9bd: 0x000a, 0x9be: 0x000a, + // Block 0x27, offset 0x9c0 + 0x9cc: 0x000c, 0x9cd: 0x000c, + 0x9e2: 0x000c, 0x9e3: 0x000c, + // Block 0x28, offset 0xa00 + 0xa01: 0x000c, + // Block 0x29, offset 0xa40 + 0xa41: 0x000c, 0xa42: 0x000c, 0xa43: 0x000c, 0xa44: 0x000c, + 0xa4d: 0x000c, + 0xa62: 0x000c, 0xa63: 0x000c, + // Block 0x2a, offset 0xa80 + 0xa8a: 0x000c, + 0xa92: 0x000c, 0xa93: 0x000c, 0xa94: 0x000c, 0xa96: 0x000c, + // Block 0x2b, offset 0xac0 + 0xaf1: 0x000c, 0xaf4: 0x000c, 0xaf5: 0x000c, + 0xaf6: 0x000c, 0xaf7: 0x000c, 0xaf8: 0x000c, 0xaf9: 0x000c, 0xafa: 0x000c, + 0xaff: 0x0004, + // Block 0x2c, offset 0xb00 + 0xb07: 0x000c, 0xb08: 0x000c, 0xb09: 0x000c, 0xb0a: 0x000c, 0xb0b: 0x000c, + 0xb0c: 0x000c, 0xb0d: 0x000c, 0xb0e: 0x000c, + // Block 0x2d, offset 0xb40 + 0xb71: 0x000c, 0xb74: 0x000c, 0xb75: 0x000c, + 0xb76: 0x000c, 0xb77: 0x000c, 0xb78: 0x000c, 0xb79: 0x000c, 0xb7b: 0x000c, + 0xb7c: 0x000c, + // Block 0x2e, offset 0xb80 + 0xb88: 0x000c, 0xb89: 0x000c, 0xb8a: 0x000c, 0xb8b: 0x000c, + 0xb8c: 0x000c, 0xb8d: 0x000c, + // Block 0x2f, offset 0xbc0 + 0xbd8: 0x000c, 0xbd9: 0x000c, + 0xbf5: 0x000c, + 0xbf7: 0x000c, 0xbf9: 0x000c, 0xbfa: 0x003a, 0xbfb: 0x002a, + 0xbfc: 0x003a, 0xbfd: 0x002a, + // Block 0x30, offset 0xc00 + 0xc31: 0x000c, 0xc32: 0x000c, 0xc33: 0x000c, 0xc34: 0x000c, 0xc35: 0x000c, + 0xc36: 0x000c, 0xc37: 0x000c, 0xc38: 0x000c, 0xc39: 0x000c, 0xc3a: 0x000c, 0xc3b: 0x000c, + 0xc3c: 0x000c, 0xc3d: 0x000c, 0xc3e: 0x000c, + // Block 0x31, offset 0xc40 + 0xc40: 0x000c, 0xc41: 0x000c, 0xc42: 0x000c, 0xc43: 0x000c, 0xc44: 0x000c, + 0xc46: 0x000c, 0xc47: 0x000c, + 0xc4d: 0x000c, 0xc4e: 0x000c, 0xc4f: 0x000c, 0xc50: 0x000c, 0xc51: 0x000c, + 0xc52: 0x000c, 0xc53: 0x000c, 0xc54: 0x000c, 0xc55: 0x000c, 0xc56: 0x000c, 0xc57: 0x000c, + 0xc59: 0x000c, 0xc5a: 0x000c, 0xc5b: 0x000c, 0xc5c: 0x000c, 0xc5d: 0x000c, + 0xc5e: 0x000c, 0xc5f: 0x000c, 0xc60: 0x000c, 0xc61: 0x000c, 0xc62: 0x000c, 0xc63: 0x000c, + 0xc64: 0x000c, 0xc65: 0x000c, 0xc66: 0x000c, 0xc67: 0x000c, 0xc68: 0x000c, 0xc69: 0x000c, + 0xc6a: 0x000c, 0xc6b: 0x000c, 0xc6c: 0x000c, 0xc6d: 0x000c, 0xc6e: 0x000c, 0xc6f: 0x000c, + 0xc70: 0x000c, 0xc71: 0x000c, 0xc72: 0x000c, 0xc73: 0x000c, 0xc74: 0x000c, 0xc75: 0x000c, + 0xc76: 0x000c, 0xc77: 0x000c, 0xc78: 0x000c, 0xc79: 0x000c, 0xc7a: 0x000c, 0xc7b: 0x000c, + 0xc7c: 0x000c, + // Block 0x32, offset 0xc80 + 0xc86: 0x000c, + // Block 0x33, offset 0xcc0 + 0xced: 0x000c, 0xcee: 0x000c, 0xcef: 0x000c, + 0xcf0: 0x000c, 0xcf2: 0x000c, 0xcf3: 0x000c, 0xcf4: 0x000c, 0xcf5: 0x000c, + 0xcf6: 0x000c, 0xcf7: 0x000c, 0xcf9: 0x000c, 0xcfa: 0x000c, + 0xcfd: 0x000c, 0xcfe: 0x000c, + // Block 0x34, offset 0xd00 + 0xd18: 0x000c, 0xd19: 0x000c, + 0xd1e: 0x000c, 0xd1f: 0x000c, 0xd20: 0x000c, + 0xd31: 0x000c, 0xd32: 0x000c, 0xd33: 0x000c, 0xd34: 0x000c, + // Block 0x35, offset 0xd40 + 0xd42: 0x000c, 0xd45: 0x000c, + 0xd46: 0x000c, + 0xd4d: 0x000c, + 0xd5d: 0x000c, + // Block 0x36, offset 0xd80 + 0xd9d: 0x000c, + 0xd9e: 0x000c, 0xd9f: 0x000c, + // Block 0x37, offset 0xdc0 + 0xdd0: 0x000a, 0xdd1: 0x000a, + 0xdd2: 0x000a, 0xdd3: 0x000a, 0xdd4: 0x000a, 0xdd5: 0x000a, 0xdd6: 0x000a, 0xdd7: 0x000a, + 0xdd8: 0x000a, 0xdd9: 0x000a, + // Block 0x38, offset 0xe00 + 0xe00: 0x000a, + // Block 0x39, offset 0xe40 + 0xe40: 0x0009, + 0xe5b: 0x007a, 0xe5c: 0x006a, + // Block 0x3a, offset 0xe80 + 0xe92: 0x000c, 0xe93: 0x000c, 0xe94: 0x000c, + 0xeb2: 0x000c, 0xeb3: 0x000c, 0xeb4: 0x000c, + // Block 0x3b, offset 0xec0 + 0xed2: 0x000c, 0xed3: 0x000c, + 0xef2: 0x000c, 0xef3: 0x000c, + // Block 0x3c, offset 0xf00 + 0xf34: 0x000c, 0xf35: 0x000c, + 0xf37: 0x000c, 0xf38: 0x000c, 0xf39: 0x000c, 0xf3a: 0x000c, 0xf3b: 0x000c, + 0xf3c: 0x000c, 0xf3d: 0x000c, + // Block 0x3d, offset 0xf40 + 0xf46: 0x000c, 0xf49: 0x000c, 0xf4a: 0x000c, 0xf4b: 0x000c, + 0xf4c: 0x000c, 0xf4d: 0x000c, 0xf4e: 0x000c, 0xf4f: 0x000c, 0xf50: 0x000c, 0xf51: 0x000c, + 0xf52: 0x000c, 0xf53: 0x000c, + 0xf5b: 0x0004, 0xf5d: 0x000c, + 0xf70: 0x000a, 0xf71: 0x000a, 0xf72: 0x000a, 0xf73: 0x000a, 0xf74: 0x000a, 0xf75: 0x000a, + 0xf76: 0x000a, 0xf77: 0x000a, 0xf78: 0x000a, 0xf79: 0x000a, + // Block 0x3e, offset 0xf80 + 0xf80: 0x000a, 0xf81: 0x000a, 0xf82: 0x000a, 0xf83: 0x000a, 0xf84: 0x000a, 0xf85: 0x000a, + 0xf86: 0x000a, 0xf87: 0x000a, 0xf88: 0x000a, 0xf89: 0x000a, 0xf8a: 0x000a, 0xf8b: 0x000c, + 0xf8c: 0x000c, 0xf8d: 0x000c, 0xf8e: 0x000b, + // Block 0x3f, offset 0xfc0 + 0xfc5: 0x000c, + 0xfc6: 0x000c, + 0xfe9: 0x000c, + // Block 0x40, offset 0x1000 + 0x1020: 0x000c, 0x1021: 0x000c, 0x1022: 0x000c, + 0x1027: 0x000c, 0x1028: 0x000c, + 0x1032: 0x000c, + 0x1039: 0x000c, 0x103a: 0x000c, 0x103b: 0x000c, + // Block 0x41, offset 0x1040 + 0x1040: 0x000a, 0x1044: 0x000a, 0x1045: 0x000a, + // Block 0x42, offset 0x1080 + 0x109e: 0x000a, 0x109f: 0x000a, 0x10a0: 0x000a, 0x10a1: 0x000a, 0x10a2: 0x000a, 0x10a3: 0x000a, + 0x10a4: 0x000a, 0x10a5: 0x000a, 0x10a6: 0x000a, 0x10a7: 0x000a, 0x10a8: 0x000a, 0x10a9: 0x000a, + 0x10aa: 0x000a, 0x10ab: 0x000a, 0x10ac: 0x000a, 0x10ad: 0x000a, 0x10ae: 0x000a, 0x10af: 0x000a, + 0x10b0: 0x000a, 0x10b1: 0x000a, 0x10b2: 0x000a, 0x10b3: 0x000a, 0x10b4: 0x000a, 0x10b5: 0x000a, + 0x10b6: 0x000a, 0x10b7: 0x000a, 0x10b8: 0x000a, 0x10b9: 0x000a, 0x10ba: 0x000a, 0x10bb: 0x000a, + 0x10bc: 0x000a, 0x10bd: 0x000a, 0x10be: 0x000a, 0x10bf: 0x000a, + // Block 0x43, offset 0x10c0 + 0x10d7: 0x000c, + 0x10d8: 0x000c, 0x10db: 0x000c, + // Block 0x44, offset 0x1100 + 0x1116: 0x000c, + 0x1118: 0x000c, 0x1119: 0x000c, 0x111a: 0x000c, 0x111b: 0x000c, 0x111c: 0x000c, 0x111d: 0x000c, + 0x111e: 0x000c, 0x1120: 0x000c, 0x1122: 0x000c, + 0x1125: 0x000c, 0x1126: 0x000c, 0x1127: 0x000c, 0x1128: 0x000c, 0x1129: 0x000c, + 0x112a: 0x000c, 0x112b: 0x000c, 0x112c: 0x000c, + 0x1133: 0x000c, 0x1134: 0x000c, 0x1135: 0x000c, + 0x1136: 0x000c, 0x1137: 0x000c, 0x1138: 0x000c, 0x1139: 0x000c, 0x113a: 0x000c, 0x113b: 0x000c, + 0x113c: 0x000c, 0x113f: 0x000c, + // Block 0x45, offset 0x1140 + 0x1170: 0x000c, 0x1171: 0x000c, 0x1172: 0x000c, 0x1173: 0x000c, 0x1174: 0x000c, 0x1175: 0x000c, + 0x1176: 0x000c, 0x1177: 0x000c, 0x1178: 0x000c, 0x1179: 0x000c, 0x117a: 0x000c, 0x117b: 0x000c, + 0x117c: 0x000c, 0x117d: 0x000c, 0x117e: 0x000c, + // Block 0x46, offset 0x1180 + 0x1180: 0x000c, 0x1181: 0x000c, 0x1182: 0x000c, 0x1183: 0x000c, + 0x11b4: 0x000c, + 0x11b6: 0x000c, 0x11b7: 0x000c, 0x11b8: 0x000c, 0x11b9: 0x000c, 0x11ba: 0x000c, + 0x11bc: 0x000c, + // Block 0x47, offset 0x11c0 + 0x11c2: 0x000c, + 0x11eb: 0x000c, 0x11ec: 0x000c, 0x11ed: 0x000c, 0x11ee: 0x000c, 0x11ef: 0x000c, + 0x11f0: 0x000c, 0x11f1: 0x000c, 0x11f2: 0x000c, 0x11f3: 0x000c, + // Block 0x48, offset 0x1200 + 0x1200: 0x000c, 0x1201: 0x000c, + 0x1222: 0x000c, 0x1223: 0x000c, + 0x1224: 0x000c, 0x1225: 0x000c, 0x1228: 0x000c, 0x1229: 0x000c, + 0x122b: 0x000c, 0x122c: 0x000c, 0x122d: 0x000c, + // Block 0x49, offset 0x1240 + 0x1266: 0x000c, 0x1268: 0x000c, 0x1269: 0x000c, + 0x126d: 0x000c, 0x126f: 0x000c, + 0x1270: 0x000c, 0x1271: 0x000c, + // Block 0x4a, offset 0x1280 + 0x12ac: 0x000c, 0x12ad: 0x000c, 0x12ae: 0x000c, 0x12af: 0x000c, + 0x12b0: 0x000c, 0x12b1: 0x000c, 0x12b2: 0x000c, 0x12b3: 0x000c, + 0x12b6: 0x000c, 0x12b7: 0x000c, + // Block 0x4b, offset 0x12c0 + 0x12d0: 0x000c, 0x12d1: 0x000c, + 0x12d2: 0x000c, 0x12d4: 0x000c, 0x12d5: 0x000c, 0x12d6: 0x000c, 0x12d7: 0x000c, + 0x12d8: 0x000c, 0x12d9: 0x000c, 0x12da: 0x000c, 0x12db: 0x000c, 0x12dc: 0x000c, 0x12dd: 0x000c, + 0x12de: 0x000c, 0x12df: 0x000c, 0x12e0: 0x000c, 0x12e2: 0x000c, 0x12e3: 0x000c, + 0x12e4: 0x000c, 0x12e5: 0x000c, 0x12e6: 0x000c, 0x12e7: 0x000c, 0x12e8: 0x000c, + 0x12ed: 0x000c, + 0x12f4: 0x000c, + 0x12f8: 0x000c, 0x12f9: 0x000c, + // Block 0x4c, offset 0x1300 + 0x1300: 0x000c, 0x1301: 0x000c, 0x1302: 0x000c, 0x1303: 0x000c, 0x1304: 0x000c, 0x1305: 0x000c, + 0x1306: 0x000c, 0x1307: 0x000c, 0x1308: 0x000c, 0x1309: 0x000c, 0x130a: 0x000c, 0x130b: 0x000c, + 0x130c: 0x000c, 0x130d: 0x000c, 0x130e: 0x000c, 0x130f: 0x000c, 0x1310: 0x000c, 0x1311: 0x000c, + 0x1312: 0x000c, 0x1313: 0x000c, 0x1314: 0x000c, 0x1315: 0x000c, 0x1316: 0x000c, 0x1317: 0x000c, + 0x1318: 0x000c, 0x1319: 0x000c, 0x131a: 0x000c, 0x131b: 0x000c, 0x131c: 0x000c, 0x131d: 0x000c, + 0x131e: 0x000c, 0x131f: 0x000c, 0x1320: 0x000c, 0x1321: 0x000c, 0x1322: 0x000c, 0x1323: 0x000c, + 0x1324: 0x000c, 0x1325: 0x000c, 0x1326: 0x000c, 0x1327: 0x000c, 0x1328: 0x000c, 0x1329: 0x000c, + 0x132a: 0x000c, 0x132b: 0x000c, 0x132c: 0x000c, 0x132d: 0x000c, 0x132e: 0x000c, 0x132f: 0x000c, + 0x1330: 0x000c, 0x1331: 0x000c, 0x1332: 0x000c, 0x1333: 0x000c, 0x1334: 0x000c, 0x1335: 0x000c, + 0x133b: 0x000c, + 0x133c: 0x000c, 0x133d: 0x000c, 0x133e: 0x000c, 0x133f: 0x000c, + // Block 0x4d, offset 0x1340 + 0x137d: 0x000a, 0x137f: 0x000a, + // Block 0x4e, offset 0x1380 + 0x1380: 0x000a, 0x1381: 0x000a, + 0x138d: 0x000a, 0x138e: 0x000a, 0x138f: 0x000a, + 0x139d: 0x000a, + 0x139e: 0x000a, 0x139f: 0x000a, + 0x13ad: 0x000a, 0x13ae: 0x000a, 0x13af: 0x000a, + 0x13bd: 0x000a, 0x13be: 0x000a, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x0009, 0x13c1: 0x0009, 0x13c2: 0x0009, 0x13c3: 0x0009, 0x13c4: 0x0009, 0x13c5: 0x0009, + 0x13c6: 0x0009, 0x13c7: 0x0009, 0x13c8: 0x0009, 0x13c9: 0x0009, 0x13ca: 0x0009, 0x13cb: 0x000b, + 0x13cc: 0x000b, 0x13cd: 0x000b, 0x13cf: 0x0001, 0x13d0: 0x000a, 0x13d1: 0x000a, + 0x13d2: 0x000a, 0x13d3: 0x000a, 0x13d4: 0x000a, 0x13d5: 0x000a, 0x13d6: 0x000a, 0x13d7: 0x000a, + 0x13d8: 0x000a, 0x13d9: 0x000a, 0x13da: 0x000a, 0x13db: 0x000a, 0x13dc: 0x000a, 0x13dd: 0x000a, + 0x13de: 0x000a, 0x13df: 0x000a, 0x13e0: 0x000a, 0x13e1: 0x000a, 0x13e2: 0x000a, 0x13e3: 0x000a, + 0x13e4: 0x000a, 0x13e5: 0x000a, 0x13e6: 0x000a, 0x13e7: 0x000a, 0x13e8: 0x0009, 0x13e9: 0x0007, + 0x13ea: 0x000e, 0x13eb: 0x000e, 0x13ec: 0x000e, 0x13ed: 0x000e, 0x13ee: 0x000e, 0x13ef: 0x0006, + 0x13f0: 0x0004, 0x13f1: 0x0004, 0x13f2: 0x0004, 0x13f3: 0x0004, 0x13f4: 0x0004, 0x13f5: 0x000a, + 0x13f6: 0x000a, 0x13f7: 0x000a, 0x13f8: 0x000a, 0x13f9: 0x000a, 0x13fa: 0x000a, 0x13fb: 0x000a, + 0x13fc: 0x000a, 0x13fd: 0x000a, 0x13fe: 0x000a, 0x13ff: 0x000a, + // Block 0x50, offset 0x1400 + 0x1400: 0x000a, 0x1401: 0x000a, 0x1402: 0x000a, 0x1403: 0x000a, 0x1404: 0x0006, 0x1405: 0x009a, + 0x1406: 0x008a, 0x1407: 0x000a, 0x1408: 0x000a, 0x1409: 0x000a, 0x140a: 0x000a, 0x140b: 0x000a, + 0x140c: 0x000a, 0x140d: 0x000a, 0x140e: 0x000a, 0x140f: 0x000a, 0x1410: 0x000a, 0x1411: 0x000a, + 0x1412: 0x000a, 0x1413: 0x000a, 0x1414: 0x000a, 0x1415: 0x000a, 0x1416: 0x000a, 0x1417: 0x000a, + 0x1418: 0x000a, 0x1419: 0x000a, 0x141a: 0x000a, 0x141b: 0x000a, 0x141c: 0x000a, 0x141d: 0x000a, + 0x141e: 0x000a, 0x141f: 0x0009, 0x1420: 0x000b, 0x1421: 0x000b, 0x1422: 0x000b, 0x1423: 0x000b, + 0x1424: 0x000b, 0x1425: 0x000b, 0x1426: 0x000e, 0x1427: 0x000e, 0x1428: 0x000e, 0x1429: 0x000e, + 0x142a: 0x000b, 0x142b: 0x000b, 0x142c: 0x000b, 0x142d: 0x000b, 0x142e: 0x000b, 0x142f: 0x000b, + 0x1430: 0x0002, 0x1434: 0x0002, 0x1435: 0x0002, + 0x1436: 0x0002, 0x1437: 0x0002, 0x1438: 0x0002, 0x1439: 0x0002, 0x143a: 0x0003, 0x143b: 0x0003, + 0x143c: 0x000a, 0x143d: 0x009a, 0x143e: 0x008a, + // Block 0x51, offset 0x1440 + 0x1440: 0x0002, 0x1441: 0x0002, 0x1442: 0x0002, 0x1443: 0x0002, 0x1444: 0x0002, 0x1445: 0x0002, + 0x1446: 0x0002, 0x1447: 0x0002, 0x1448: 0x0002, 0x1449: 0x0002, 0x144a: 0x0003, 0x144b: 0x0003, + 0x144c: 0x000a, 0x144d: 0x009a, 0x144e: 0x008a, + 0x1460: 0x0004, 0x1461: 0x0004, 0x1462: 0x0004, 0x1463: 0x0004, + 0x1464: 0x0004, 0x1465: 0x0004, 0x1466: 0x0004, 0x1467: 0x0004, 0x1468: 0x0004, 0x1469: 0x0004, + 0x146a: 0x0004, 0x146b: 0x0004, 0x146c: 0x0004, 0x146d: 0x0004, 0x146e: 0x0004, 0x146f: 0x0004, + 0x1470: 0x0004, 0x1471: 0x0004, 0x1472: 0x0004, 0x1473: 0x0004, 0x1474: 0x0004, 0x1475: 0x0004, + 0x1476: 0x0004, 0x1477: 0x0004, 0x1478: 0x0004, 0x1479: 0x0004, 0x147a: 0x0004, 0x147b: 0x0004, + 0x147c: 0x0004, 0x147d: 0x0004, 0x147e: 0x0004, 0x147f: 0x0004, + // Block 0x52, offset 0x1480 + 0x1480: 0x0004, 0x1481: 0x0004, 0x1482: 0x0004, 0x1483: 0x0004, 0x1484: 0x0004, 0x1485: 0x0004, + 0x1486: 0x0004, 0x1487: 0x0004, 0x1488: 0x0004, 0x1489: 0x0004, 0x148a: 0x0004, 0x148b: 0x0004, + 0x148c: 0x0004, 0x148d: 0x0004, 0x148e: 0x0004, 0x148f: 0x0004, 0x1490: 0x000c, 0x1491: 0x000c, + 0x1492: 0x000c, 0x1493: 0x000c, 0x1494: 0x000c, 0x1495: 0x000c, 0x1496: 0x000c, 0x1497: 0x000c, + 0x1498: 0x000c, 0x1499: 0x000c, 0x149a: 0x000c, 0x149b: 0x000c, 0x149c: 0x000c, 0x149d: 0x000c, + 0x149e: 0x000c, 0x149f: 0x000c, 0x14a0: 0x000c, 0x14a1: 0x000c, 0x14a2: 0x000c, 0x14a3: 0x000c, + 0x14a4: 0x000c, 0x14a5: 0x000c, 0x14a6: 0x000c, 0x14a7: 0x000c, 0x14a8: 0x000c, 0x14a9: 0x000c, + 0x14aa: 0x000c, 0x14ab: 0x000c, 0x14ac: 0x000c, 0x14ad: 0x000c, 0x14ae: 0x000c, 0x14af: 0x000c, + 0x14b0: 0x000c, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x000a, 0x14c1: 0x000a, 0x14c3: 0x000a, 0x14c4: 0x000a, 0x14c5: 0x000a, + 0x14c6: 0x000a, 0x14c8: 0x000a, 0x14c9: 0x000a, + 0x14d4: 0x000a, 0x14d6: 0x000a, 0x14d7: 0x000a, + 0x14d8: 0x000a, + 0x14de: 0x000a, 0x14df: 0x000a, 0x14e0: 0x000a, 0x14e1: 0x000a, 0x14e2: 0x000a, 0x14e3: 0x000a, + 0x14e5: 0x000a, 0x14e7: 0x000a, 0x14e9: 0x000a, + 0x14ee: 0x0004, + 0x14fa: 0x000a, 0x14fb: 0x000a, + // Block 0x54, offset 0x1500 + 0x1500: 0x000a, 0x1501: 0x000a, 0x1502: 0x000a, 0x1503: 0x000a, 0x1504: 0x000a, + 0x150a: 0x000a, 0x150b: 0x000a, + 0x150c: 0x000a, 0x150d: 0x000a, 0x1510: 0x000a, 0x1511: 0x000a, + 0x1512: 0x000a, 0x1513: 0x000a, 0x1514: 0x000a, 0x1515: 0x000a, 0x1516: 0x000a, 0x1517: 0x000a, + 0x1518: 0x000a, 0x1519: 0x000a, 0x151a: 0x000a, 0x151b: 0x000a, 0x151c: 0x000a, 0x151d: 0x000a, + 0x151e: 0x000a, 0x151f: 0x000a, + // Block 0x55, offset 0x1540 + 0x1549: 0x000a, 0x154a: 0x000a, 0x154b: 0x000a, + 0x1550: 0x000a, 0x1551: 0x000a, + 0x1552: 0x000a, 0x1553: 0x000a, 0x1554: 0x000a, 0x1555: 0x000a, 0x1556: 0x000a, 0x1557: 0x000a, + 0x1558: 0x000a, 0x1559: 0x000a, 0x155a: 0x000a, 0x155b: 0x000a, 0x155c: 0x000a, 0x155d: 0x000a, + 0x155e: 0x000a, 0x155f: 0x000a, 0x1560: 0x000a, 0x1561: 0x000a, 0x1562: 0x000a, 0x1563: 0x000a, + 0x1564: 0x000a, 0x1565: 0x000a, 0x1566: 0x000a, 0x1567: 0x000a, 0x1568: 0x000a, 0x1569: 0x000a, + 0x156a: 0x000a, 0x156b: 0x000a, 0x156c: 0x000a, 0x156d: 0x000a, 0x156e: 0x000a, 0x156f: 0x000a, + 0x1570: 0x000a, 0x1571: 0x000a, 0x1572: 0x000a, 0x1573: 0x000a, 0x1574: 0x000a, 0x1575: 0x000a, + 0x1576: 0x000a, 0x1577: 0x000a, 0x1578: 0x000a, 0x1579: 0x000a, 0x157a: 0x000a, 0x157b: 0x000a, + 0x157c: 0x000a, 0x157d: 0x000a, 0x157e: 0x000a, 0x157f: 0x000a, + // Block 0x56, offset 0x1580 + 0x1580: 0x000a, 0x1581: 0x000a, 0x1582: 0x000a, 0x1583: 0x000a, 0x1584: 0x000a, 0x1585: 0x000a, + 0x1586: 0x000a, 0x1587: 0x000a, 0x1588: 0x000a, 0x1589: 0x000a, 0x158a: 0x000a, 0x158b: 0x000a, + 0x158c: 0x000a, 0x158d: 0x000a, 0x158e: 0x000a, 0x158f: 0x000a, 0x1590: 0x000a, 0x1591: 0x000a, + 0x1592: 0x000a, 0x1593: 0x000a, 0x1594: 0x000a, 0x1595: 0x000a, 0x1596: 0x000a, 0x1597: 0x000a, + 0x1598: 0x000a, 0x1599: 0x000a, 0x159a: 0x000a, 0x159b: 0x000a, 0x159c: 0x000a, 0x159d: 0x000a, + 0x159e: 0x000a, 0x159f: 0x000a, 0x15a0: 0x000a, 0x15a1: 0x000a, 0x15a2: 0x000a, 0x15a3: 0x000a, + 0x15a4: 0x000a, 0x15a5: 0x000a, 0x15a6: 0x000a, 0x15a7: 0x000a, 0x15a8: 0x000a, 0x15a9: 0x000a, + 0x15aa: 0x000a, 0x15ab: 0x000a, 0x15ac: 0x000a, 0x15ad: 0x000a, 0x15ae: 0x000a, 0x15af: 0x000a, + 0x15b0: 0x000a, 0x15b1: 0x000a, 0x15b2: 0x000a, 0x15b3: 0x000a, 0x15b4: 0x000a, 0x15b5: 0x000a, + 0x15b6: 0x000a, 0x15b7: 0x000a, 0x15b8: 0x000a, 0x15b9: 0x000a, 0x15ba: 0x000a, 0x15bb: 0x000a, + 0x15bc: 0x000a, 0x15bd: 0x000a, 0x15be: 0x000a, 0x15bf: 0x000a, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x000a, 0x15c1: 0x000a, 0x15c2: 0x000a, 0x15c3: 0x000a, 0x15c4: 0x000a, 0x15c5: 0x000a, + 0x15c6: 0x000a, 0x15c7: 0x000a, 0x15c8: 0x000a, 0x15c9: 0x000a, 0x15ca: 0x000a, 0x15cb: 0x000a, + 0x15cc: 0x000a, 0x15cd: 0x000a, 0x15ce: 0x000a, 0x15cf: 0x000a, 0x15d0: 0x000a, 0x15d1: 0x000a, + 0x15d2: 0x0003, 0x15d3: 0x0004, 0x15d4: 0x000a, 0x15d5: 0x000a, 0x15d6: 0x000a, 0x15d7: 0x000a, + 0x15d8: 0x000a, 0x15d9: 0x000a, 0x15da: 0x000a, 0x15db: 0x000a, 0x15dc: 0x000a, 0x15dd: 0x000a, + 0x15de: 0x000a, 0x15df: 0x000a, 0x15e0: 0x000a, 0x15e1: 0x000a, 0x15e2: 0x000a, 0x15e3: 0x000a, + 0x15e4: 0x000a, 0x15e5: 0x000a, 0x15e6: 0x000a, 0x15e7: 0x000a, 0x15e8: 0x000a, 0x15e9: 0x000a, + 0x15ea: 0x000a, 0x15eb: 0x000a, 0x15ec: 0x000a, 0x15ed: 0x000a, 0x15ee: 0x000a, 0x15ef: 0x000a, + 0x15f0: 0x000a, 0x15f1: 0x000a, 0x15f2: 0x000a, 0x15f3: 0x000a, 0x15f4: 0x000a, 0x15f5: 0x000a, + 0x15f6: 0x000a, 0x15f7: 0x000a, 0x15f8: 0x000a, 0x15f9: 0x000a, 0x15fa: 0x000a, 0x15fb: 0x000a, + 0x15fc: 0x000a, 0x15fd: 0x000a, 0x15fe: 0x000a, 0x15ff: 0x000a, + // Block 0x58, offset 0x1600 + 0x1600: 0x000a, 0x1601: 0x000a, 0x1602: 0x000a, 0x1603: 0x000a, 0x1604: 0x000a, 0x1605: 0x000a, + 0x1606: 0x000a, 0x1607: 0x000a, 0x1608: 0x003a, 0x1609: 0x002a, 0x160a: 0x003a, 0x160b: 0x002a, + 0x160c: 0x000a, 0x160d: 0x000a, 0x160e: 0x000a, 0x160f: 0x000a, 0x1610: 0x000a, 0x1611: 0x000a, + 0x1612: 0x000a, 0x1613: 0x000a, 0x1614: 0x000a, 0x1615: 0x000a, 0x1616: 0x000a, 0x1617: 0x000a, + 0x1618: 0x000a, 0x1619: 0x000a, 0x161a: 0x000a, 0x161b: 0x000a, 0x161c: 0x000a, 0x161d: 0x000a, + 0x161e: 0x000a, 0x161f: 0x000a, 0x1620: 0x000a, 0x1621: 0x000a, 0x1622: 0x000a, 0x1623: 0x000a, + 0x1624: 0x000a, 0x1625: 0x000a, 0x1626: 0x000a, 0x1627: 0x000a, 0x1628: 0x000a, 0x1629: 0x009a, + 0x162a: 0x008a, 0x162b: 0x000a, 0x162c: 0x000a, 0x162d: 0x000a, 0x162e: 0x000a, 0x162f: 0x000a, + 0x1630: 0x000a, 0x1631: 0x000a, 0x1632: 0x000a, 0x1633: 0x000a, 0x1634: 0x000a, 0x1635: 0x000a, + // Block 0x59, offset 0x1640 + 0x167b: 0x000a, + 0x167c: 0x000a, 0x167d: 0x000a, 0x167e: 0x000a, 0x167f: 0x000a, + // Block 0x5a, offset 0x1680 + 0x1680: 0x000a, 0x1681: 0x000a, 0x1682: 0x000a, 0x1683: 0x000a, 0x1684: 0x000a, 0x1685: 0x000a, + 0x1686: 0x000a, 0x1687: 0x000a, 0x1688: 0x000a, 0x1689: 0x000a, 0x168a: 0x000a, 0x168b: 0x000a, + 0x168c: 0x000a, 0x168d: 0x000a, 0x168e: 0x000a, 0x168f: 0x000a, 0x1690: 0x000a, 0x1691: 0x000a, + 0x1692: 0x000a, 0x1693: 0x000a, 0x1694: 0x000a, 0x1696: 0x000a, 0x1697: 0x000a, + 0x1698: 0x000a, 0x1699: 0x000a, 0x169a: 0x000a, 0x169b: 0x000a, 0x169c: 0x000a, 0x169d: 0x000a, + 0x169e: 0x000a, 0x169f: 0x000a, 0x16a0: 0x000a, 0x16a1: 0x000a, 0x16a2: 0x000a, 0x16a3: 0x000a, + 0x16a4: 0x000a, 0x16a5: 0x000a, 0x16a6: 0x000a, 0x16a7: 0x000a, 0x16a8: 0x000a, 0x16a9: 0x000a, + 0x16aa: 0x000a, 0x16ab: 0x000a, 0x16ac: 0x000a, 0x16ad: 0x000a, 0x16ae: 0x000a, 0x16af: 0x000a, + 0x16b0: 0x000a, 0x16b1: 0x000a, 0x16b2: 0x000a, 0x16b3: 0x000a, 0x16b4: 0x000a, 0x16b5: 0x000a, + 0x16b6: 0x000a, 0x16b7: 0x000a, 0x16b8: 0x000a, 0x16b9: 0x000a, 0x16ba: 0x000a, 0x16bb: 0x000a, + 0x16bc: 0x000a, 0x16bd: 0x000a, 0x16be: 0x000a, 0x16bf: 0x000a, + // Block 0x5b, offset 0x16c0 + 0x16c0: 0x000a, 0x16c1: 0x000a, 0x16c2: 0x000a, 0x16c3: 0x000a, 0x16c4: 0x000a, 0x16c5: 0x000a, + 0x16c6: 0x000a, 0x16c7: 0x000a, 0x16c8: 0x000a, 0x16c9: 0x000a, 0x16ca: 0x000a, 0x16cb: 0x000a, + 0x16cc: 0x000a, 0x16cd: 0x000a, 0x16ce: 0x000a, 0x16cf: 0x000a, 0x16d0: 0x000a, 0x16d1: 0x000a, + 0x16d2: 0x000a, 0x16d3: 0x000a, 0x16d4: 0x000a, 0x16d5: 0x000a, 0x16d6: 0x000a, 0x16d7: 0x000a, + 0x16d8: 0x000a, 0x16d9: 0x000a, 0x16da: 0x000a, 0x16db: 0x000a, 0x16dc: 0x000a, 0x16dd: 0x000a, + 0x16de: 0x000a, 0x16df: 0x000a, 0x16e0: 0x000a, 0x16e1: 0x000a, 0x16e2: 0x000a, 0x16e3: 0x000a, + 0x16e4: 0x000a, 0x16e5: 0x000a, 0x16e6: 0x000a, 0x16e7: 0x000a, 0x16e8: 0x000a, 0x16e9: 0x000a, + 0x16ea: 0x000a, 0x16eb: 0x000a, 0x16ec: 0x000a, 0x16ed: 0x000a, 0x16ee: 0x000a, 0x16ef: 0x000a, + 0x16f0: 0x000a, 0x16f1: 0x000a, 0x16f2: 0x000a, 0x16f3: 0x000a, 0x16f4: 0x000a, 0x16f5: 0x000a, + 0x16f6: 0x000a, 0x16f7: 0x000a, 0x16f8: 0x000a, 0x16f9: 0x000a, 0x16fa: 0x000a, 0x16fb: 0x000a, + 0x16fc: 0x000a, 0x16fd: 0x000a, 0x16fe: 0x000a, + // Block 0x5c, offset 0x1700 + 0x1700: 0x000a, 0x1701: 0x000a, 0x1702: 0x000a, 0x1703: 0x000a, 0x1704: 0x000a, 0x1705: 0x000a, + 0x1706: 0x000a, 0x1707: 0x000a, 0x1708: 0x000a, 0x1709: 0x000a, 0x170a: 0x000a, 0x170b: 0x000a, + 0x170c: 0x000a, 0x170d: 0x000a, 0x170e: 0x000a, 0x170f: 0x000a, 0x1710: 0x000a, 0x1711: 0x000a, + 0x1712: 0x000a, 0x1713: 0x000a, 0x1714: 0x000a, 0x1715: 0x000a, 0x1716: 0x000a, 0x1717: 0x000a, + 0x1718: 0x000a, 0x1719: 0x000a, 0x171a: 0x000a, 0x171b: 0x000a, 0x171c: 0x000a, 0x171d: 0x000a, + 0x171e: 0x000a, 0x171f: 0x000a, 0x1720: 0x000a, 0x1721: 0x000a, 0x1722: 0x000a, 0x1723: 0x000a, + 0x1724: 0x000a, 0x1725: 0x000a, 0x1726: 0x000a, + // Block 0x5d, offset 0x1740 + 0x1740: 0x000a, 0x1741: 0x000a, 0x1742: 0x000a, 0x1743: 0x000a, 0x1744: 0x000a, 0x1745: 0x000a, + 0x1746: 0x000a, 0x1747: 0x000a, 0x1748: 0x000a, 0x1749: 0x000a, 0x174a: 0x000a, + 0x1760: 0x000a, 0x1761: 0x000a, 0x1762: 0x000a, 0x1763: 0x000a, + 0x1764: 0x000a, 0x1765: 0x000a, 0x1766: 0x000a, 0x1767: 0x000a, 0x1768: 0x000a, 0x1769: 0x000a, + 0x176a: 0x000a, 0x176b: 0x000a, 0x176c: 0x000a, 0x176d: 0x000a, 0x176e: 0x000a, 0x176f: 0x000a, + 0x1770: 0x000a, 0x1771: 0x000a, 0x1772: 0x000a, 0x1773: 0x000a, 0x1774: 0x000a, 0x1775: 0x000a, + 0x1776: 0x000a, 0x1777: 0x000a, 0x1778: 0x000a, 0x1779: 0x000a, 0x177a: 0x000a, 0x177b: 0x000a, + 0x177c: 0x000a, 0x177d: 0x000a, 0x177e: 0x000a, 0x177f: 0x000a, + // Block 0x5e, offset 0x1780 + 0x1780: 0x000a, 0x1781: 0x000a, 0x1782: 0x000a, 0x1783: 0x000a, 0x1784: 0x000a, 0x1785: 0x000a, + 0x1786: 0x000a, 0x1787: 0x000a, 0x1788: 0x0002, 0x1789: 0x0002, 0x178a: 0x0002, 0x178b: 0x0002, + 0x178c: 0x0002, 0x178d: 0x0002, 0x178e: 0x0002, 0x178f: 0x0002, 0x1790: 0x0002, 0x1791: 0x0002, + 0x1792: 0x0002, 0x1793: 0x0002, 0x1794: 0x0002, 0x1795: 0x0002, 0x1796: 0x0002, 0x1797: 0x0002, + 0x1798: 0x0002, 0x1799: 0x0002, 0x179a: 0x0002, 0x179b: 0x0002, + // Block 0x5f, offset 0x17c0 + 0x17ea: 0x000a, 0x17eb: 0x000a, 0x17ec: 0x000a, 0x17ed: 0x000a, 0x17ee: 0x000a, 0x17ef: 0x000a, + 0x17f0: 0x000a, 0x17f1: 0x000a, 0x17f2: 0x000a, 0x17f3: 0x000a, 0x17f4: 0x000a, 0x17f5: 0x000a, + 0x17f6: 0x000a, 0x17f7: 0x000a, 0x17f8: 0x000a, 0x17f9: 0x000a, 0x17fa: 0x000a, 0x17fb: 0x000a, + 0x17fc: 0x000a, 0x17fd: 0x000a, 0x17fe: 0x000a, 0x17ff: 0x000a, + // Block 0x60, offset 0x1800 + 0x1800: 0x000a, 0x1801: 0x000a, 0x1802: 0x000a, 0x1803: 0x000a, 0x1804: 0x000a, 0x1805: 0x000a, + 0x1806: 0x000a, 0x1807: 0x000a, 0x1808: 0x000a, 0x1809: 0x000a, 0x180a: 0x000a, 0x180b: 0x000a, + 0x180c: 0x000a, 0x180d: 0x000a, 0x180e: 0x000a, 0x180f: 0x000a, 0x1810: 0x000a, 0x1811: 0x000a, + 0x1812: 0x000a, 0x1813: 0x000a, 0x1814: 0x000a, 0x1815: 0x000a, 0x1816: 0x000a, 0x1817: 0x000a, + 0x1818: 0x000a, 0x1819: 0x000a, 0x181a: 0x000a, 0x181b: 0x000a, 0x181c: 0x000a, 0x181d: 0x000a, + 0x181e: 0x000a, 0x181f: 0x000a, 0x1820: 0x000a, 0x1821: 0x000a, 0x1822: 0x000a, 0x1823: 0x000a, + 0x1824: 0x000a, 0x1825: 0x000a, 0x1826: 0x000a, 0x1827: 0x000a, 0x1828: 0x000a, 0x1829: 0x000a, + 0x182a: 0x000a, 0x182b: 0x000a, 0x182d: 0x000a, 0x182e: 0x000a, 0x182f: 0x000a, + 0x1830: 0x000a, 0x1831: 0x000a, 0x1832: 0x000a, 0x1833: 0x000a, 0x1834: 0x000a, 0x1835: 0x000a, + 0x1836: 0x000a, 0x1837: 0x000a, 0x1838: 0x000a, 0x1839: 0x000a, 0x183a: 0x000a, 0x183b: 0x000a, + 0x183c: 0x000a, 0x183d: 0x000a, 0x183e: 0x000a, 0x183f: 0x000a, + // Block 0x61, offset 0x1840 + 0x1840: 0x000a, 0x1841: 0x000a, 0x1842: 0x000a, 0x1843: 0x000a, 0x1844: 0x000a, 0x1845: 0x000a, + 0x1846: 0x000a, 0x1847: 0x000a, 0x1848: 0x000a, 0x1849: 0x000a, 0x184a: 0x000a, 0x184b: 0x000a, + 0x184c: 0x000a, 0x184d: 0x000a, 0x184e: 0x000a, 0x184f: 0x000a, 0x1850: 0x000a, 0x1851: 0x000a, + 0x1852: 0x000a, 0x1853: 0x000a, 0x1854: 0x000a, 0x1855: 0x000a, 0x1856: 0x000a, 0x1857: 0x000a, + 0x1858: 0x000a, 0x1859: 0x000a, 0x185a: 0x000a, 0x185b: 0x000a, 0x185c: 0x000a, 0x185d: 0x000a, + 0x185e: 0x000a, 0x185f: 0x000a, 0x1860: 0x000a, 0x1861: 0x000a, 0x1862: 0x000a, 0x1863: 0x000a, + 0x1864: 0x000a, 0x1865: 0x000a, 0x1866: 0x000a, 0x1867: 0x000a, 0x1868: 0x003a, 0x1869: 0x002a, + 0x186a: 0x003a, 0x186b: 0x002a, 0x186c: 0x003a, 0x186d: 0x002a, 0x186e: 0x003a, 0x186f: 0x002a, + 0x1870: 0x003a, 0x1871: 0x002a, 0x1872: 0x003a, 0x1873: 0x002a, 0x1874: 0x003a, 0x1875: 0x002a, + 0x1876: 0x000a, 0x1877: 0x000a, 0x1878: 0x000a, 0x1879: 0x000a, 0x187a: 0x000a, 0x187b: 0x000a, + 0x187c: 0x000a, 0x187d: 0x000a, 0x187e: 0x000a, 0x187f: 0x000a, + // Block 0x62, offset 0x1880 + 0x1880: 0x000a, 0x1881: 0x000a, 0x1882: 0x000a, 0x1883: 0x000a, 0x1884: 0x000a, 0x1885: 0x009a, + 0x1886: 0x008a, 0x1887: 0x000a, 0x1888: 0x000a, 0x1889: 0x000a, 0x188a: 0x000a, 0x188b: 0x000a, + 0x188c: 0x000a, 0x188d: 0x000a, 0x188e: 0x000a, 0x188f: 0x000a, 0x1890: 0x000a, 0x1891: 0x000a, + 0x1892: 0x000a, 0x1893: 0x000a, 0x1894: 0x000a, 0x1895: 0x000a, 0x1896: 0x000a, 0x1897: 0x000a, + 0x1898: 0x000a, 0x1899: 0x000a, 0x189a: 0x000a, 0x189b: 0x000a, 0x189c: 0x000a, 0x189d: 0x000a, + 0x189e: 0x000a, 0x189f: 0x000a, 0x18a0: 0x000a, 0x18a1: 0x000a, 0x18a2: 0x000a, 0x18a3: 0x000a, + 0x18a4: 0x000a, 0x18a5: 0x000a, 0x18a6: 0x003a, 0x18a7: 0x002a, 0x18a8: 0x003a, 0x18a9: 0x002a, + 0x18aa: 0x003a, 0x18ab: 0x002a, 0x18ac: 0x003a, 0x18ad: 0x002a, 0x18ae: 0x003a, 0x18af: 0x002a, + 0x18b0: 0x000a, 0x18b1: 0x000a, 0x18b2: 0x000a, 0x18b3: 0x000a, 0x18b4: 0x000a, 0x18b5: 0x000a, + 0x18b6: 0x000a, 0x18b7: 0x000a, 0x18b8: 0x000a, 0x18b9: 0x000a, 0x18ba: 0x000a, 0x18bb: 0x000a, + 0x18bc: 0x000a, 0x18bd: 0x000a, 0x18be: 0x000a, 0x18bf: 0x000a, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x000a, 0x18c1: 0x000a, 0x18c2: 0x000a, 0x18c3: 0x007a, 0x18c4: 0x006a, 0x18c5: 0x009a, + 0x18c6: 0x008a, 0x18c7: 0x00ba, 0x18c8: 0x00aa, 0x18c9: 0x009a, 0x18ca: 0x008a, 0x18cb: 0x007a, + 0x18cc: 0x006a, 0x18cd: 0x00da, 0x18ce: 0x002a, 0x18cf: 0x003a, 0x18d0: 0x00ca, 0x18d1: 0x009a, + 0x18d2: 0x008a, 0x18d3: 0x007a, 0x18d4: 0x006a, 0x18d5: 0x009a, 0x18d6: 0x008a, 0x18d7: 0x00ba, + 0x18d8: 0x00aa, 0x18d9: 0x000a, 0x18da: 0x000a, 0x18db: 0x000a, 0x18dc: 0x000a, 0x18dd: 0x000a, + 0x18de: 0x000a, 0x18df: 0x000a, 0x18e0: 0x000a, 0x18e1: 0x000a, 0x18e2: 0x000a, 0x18e3: 0x000a, + 0x18e4: 0x000a, 0x18e5: 0x000a, 0x18e6: 0x000a, 0x18e7: 0x000a, 0x18e8: 0x000a, 0x18e9: 0x000a, + 0x18ea: 0x000a, 0x18eb: 0x000a, 0x18ec: 0x000a, 0x18ed: 0x000a, 0x18ee: 0x000a, 0x18ef: 0x000a, + 0x18f0: 0x000a, 0x18f1: 0x000a, 0x18f2: 0x000a, 0x18f3: 0x000a, 0x18f4: 0x000a, 0x18f5: 0x000a, + 0x18f6: 0x000a, 0x18f7: 0x000a, 0x18f8: 0x000a, 0x18f9: 0x000a, 0x18fa: 0x000a, 0x18fb: 0x000a, + 0x18fc: 0x000a, 0x18fd: 0x000a, 0x18fe: 0x000a, 0x18ff: 0x000a, + // Block 0x64, offset 0x1900 + 0x1900: 0x000a, 0x1901: 0x000a, 0x1902: 0x000a, 0x1903: 0x000a, 0x1904: 0x000a, 0x1905: 0x000a, + 0x1906: 0x000a, 0x1907: 0x000a, 0x1908: 0x000a, 0x1909: 0x000a, 0x190a: 0x000a, 0x190b: 0x000a, + 0x190c: 0x000a, 0x190d: 0x000a, 0x190e: 0x000a, 0x190f: 0x000a, 0x1910: 0x000a, 0x1911: 0x000a, + 0x1912: 0x000a, 0x1913: 0x000a, 0x1914: 0x000a, 0x1915: 0x000a, 0x1916: 0x000a, 0x1917: 0x000a, + 0x1918: 0x003a, 0x1919: 0x002a, 0x191a: 0x003a, 0x191b: 0x002a, 0x191c: 0x000a, 0x191d: 0x000a, + 0x191e: 0x000a, 0x191f: 0x000a, 0x1920: 0x000a, 0x1921: 0x000a, 0x1922: 0x000a, 0x1923: 0x000a, + 0x1924: 0x000a, 0x1925: 0x000a, 0x1926: 0x000a, 0x1927: 0x000a, 0x1928: 0x000a, 0x1929: 0x000a, + 0x192a: 0x000a, 0x192b: 0x000a, 0x192c: 0x000a, 0x192d: 0x000a, 0x192e: 0x000a, 0x192f: 0x000a, + 0x1930: 0x000a, 0x1931: 0x000a, 0x1932: 0x000a, 0x1933: 0x000a, 0x1934: 0x000a, 0x1935: 0x000a, + 0x1936: 0x000a, 0x1937: 0x000a, 0x1938: 0x000a, 0x1939: 0x000a, 0x193a: 0x000a, 0x193b: 0x000a, + 0x193c: 0x003a, 0x193d: 0x002a, 0x193e: 0x000a, 0x193f: 0x000a, + // Block 0x65, offset 0x1940 + 0x1940: 0x000a, 0x1941: 0x000a, 0x1942: 0x000a, 0x1943: 0x000a, 0x1944: 0x000a, 0x1945: 0x000a, + 0x1946: 0x000a, 0x1947: 0x000a, 0x1948: 0x000a, 0x1949: 0x000a, 0x194a: 0x000a, 0x194b: 0x000a, + 0x194c: 0x000a, 0x194d: 0x000a, 0x194e: 0x000a, 0x194f: 0x000a, 0x1950: 0x000a, 0x1951: 0x000a, + 0x1952: 0x000a, 0x1953: 0x000a, 0x1954: 0x000a, 0x1955: 0x000a, 0x1956: 0x000a, 0x1957: 0x000a, + 0x1958: 0x000a, 0x1959: 0x000a, 0x195a: 0x000a, 0x195b: 0x000a, 0x195c: 0x000a, 0x195d: 0x000a, + 0x195e: 0x000a, 0x195f: 0x000a, 0x1960: 0x000a, 0x1961: 0x000a, 0x1962: 0x000a, 0x1963: 0x000a, + 0x1964: 0x000a, 0x1965: 0x000a, 0x1966: 0x000a, 0x1967: 0x000a, 0x1968: 0x000a, 0x1969: 0x000a, + 0x196a: 0x000a, 0x196b: 0x000a, 0x196c: 0x000a, 0x196d: 0x000a, 0x196e: 0x000a, 0x196f: 0x000a, + 0x1970: 0x000a, 0x1971: 0x000a, 0x1972: 0x000a, 0x1973: 0x000a, + 0x1976: 0x000a, 0x1977: 0x000a, 0x1978: 0x000a, 0x1979: 0x000a, 0x197a: 0x000a, 0x197b: 0x000a, + 0x197c: 0x000a, 0x197d: 0x000a, 0x197e: 0x000a, 0x197f: 0x000a, + // Block 0x66, offset 0x1980 + 0x1980: 0x000a, 0x1981: 0x000a, 0x1982: 0x000a, 0x1983: 0x000a, 0x1984: 0x000a, 0x1985: 0x000a, + 0x1986: 0x000a, 0x1987: 0x000a, 0x1988: 0x000a, 0x1989: 0x000a, 0x198a: 0x000a, 0x198b: 0x000a, + 0x198c: 0x000a, 0x198d: 0x000a, 0x198e: 0x000a, 0x198f: 0x000a, 0x1990: 0x000a, 0x1991: 0x000a, + 0x1992: 0x000a, 0x1993: 0x000a, 0x1994: 0x000a, 0x1995: 0x000a, + 0x1998: 0x000a, 0x1999: 0x000a, 0x199a: 0x000a, 0x199b: 0x000a, 0x199c: 0x000a, 0x199d: 0x000a, + 0x199e: 0x000a, 0x199f: 0x000a, 0x19a0: 0x000a, 0x19a1: 0x000a, 0x19a2: 0x000a, 0x19a3: 0x000a, + 0x19a4: 0x000a, 0x19a5: 0x000a, 0x19a6: 0x000a, 0x19a7: 0x000a, 0x19a8: 0x000a, 0x19a9: 0x000a, + 0x19aa: 0x000a, 0x19ab: 0x000a, 0x19ac: 0x000a, 0x19ad: 0x000a, 0x19ae: 0x000a, 0x19af: 0x000a, + 0x19b0: 0x000a, 0x19b1: 0x000a, 0x19b2: 0x000a, 0x19b3: 0x000a, 0x19b4: 0x000a, 0x19b5: 0x000a, + 0x19b6: 0x000a, 0x19b7: 0x000a, 0x19b8: 0x000a, 0x19b9: 0x000a, + 0x19bd: 0x000a, 0x19be: 0x000a, 0x19bf: 0x000a, + // Block 0x67, offset 0x19c0 + 0x19c0: 0x000a, 0x19c1: 0x000a, 0x19c2: 0x000a, 0x19c3: 0x000a, 0x19c4: 0x000a, 0x19c5: 0x000a, + 0x19c6: 0x000a, 0x19c7: 0x000a, 0x19c8: 0x000a, 0x19ca: 0x000a, 0x19cb: 0x000a, + 0x19cc: 0x000a, 0x19cd: 0x000a, 0x19ce: 0x000a, 0x19cf: 0x000a, 0x19d0: 0x000a, 0x19d1: 0x000a, + 0x19ec: 0x000a, 0x19ed: 0x000a, 0x19ee: 0x000a, 0x19ef: 0x000a, + // Block 0x68, offset 0x1a00 + 0x1a25: 0x000a, 0x1a26: 0x000a, 0x1a27: 0x000a, 0x1a28: 0x000a, 0x1a29: 0x000a, + 0x1a2a: 0x000a, 0x1a2f: 0x000c, + 0x1a30: 0x000c, 0x1a31: 0x000c, + 0x1a39: 0x000a, 0x1a3a: 0x000a, 0x1a3b: 0x000a, + 0x1a3c: 0x000a, 0x1a3d: 0x000a, 0x1a3e: 0x000a, 0x1a3f: 0x000a, + // Block 0x69, offset 0x1a40 + 0x1a7f: 0x000c, + // Block 0x6a, offset 0x1a80 + 0x1aa0: 0x000c, 0x1aa1: 0x000c, 0x1aa2: 0x000c, 0x1aa3: 0x000c, + 0x1aa4: 0x000c, 0x1aa5: 0x000c, 0x1aa6: 0x000c, 0x1aa7: 0x000c, 0x1aa8: 0x000c, 0x1aa9: 0x000c, + 0x1aaa: 0x000c, 0x1aab: 0x000c, 0x1aac: 0x000c, 0x1aad: 0x000c, 0x1aae: 0x000c, 0x1aaf: 0x000c, + 0x1ab0: 0x000c, 0x1ab1: 0x000c, 0x1ab2: 0x000c, 0x1ab3: 0x000c, 0x1ab4: 0x000c, 0x1ab5: 0x000c, + 0x1ab6: 0x000c, 0x1ab7: 0x000c, 0x1ab8: 0x000c, 0x1ab9: 0x000c, 0x1aba: 0x000c, 0x1abb: 0x000c, + 0x1abc: 0x000c, 0x1abd: 0x000c, 0x1abe: 0x000c, 0x1abf: 0x000c, + // Block 0x6b, offset 0x1ac0 + 0x1ac0: 0x000a, 0x1ac1: 0x000a, 0x1ac2: 0x000a, 0x1ac3: 0x000a, 0x1ac4: 0x000a, 0x1ac5: 0x000a, + 0x1ac6: 0x000a, 0x1ac7: 0x000a, 0x1ac8: 0x000a, 0x1ac9: 0x000a, 0x1aca: 0x000a, 0x1acb: 0x000a, + 0x1acc: 0x000a, 0x1acd: 0x000a, 0x1ace: 0x000a, 0x1acf: 0x000a, 0x1ad0: 0x000a, 0x1ad1: 0x000a, + 0x1ad2: 0x000a, 0x1ad3: 0x000a, 0x1ad4: 0x000a, 0x1ad5: 0x000a, 0x1ad6: 0x000a, 0x1ad7: 0x000a, + 0x1ad8: 0x000a, 0x1ad9: 0x000a, 0x1ada: 0x000a, 0x1adb: 0x000a, 0x1adc: 0x000a, 0x1add: 0x000a, + 0x1ade: 0x000a, 0x1adf: 0x000a, 0x1ae0: 0x000a, 0x1ae1: 0x000a, 0x1ae2: 0x003a, 0x1ae3: 0x002a, + 0x1ae4: 0x003a, 0x1ae5: 0x002a, 0x1ae6: 0x003a, 0x1ae7: 0x002a, 0x1ae8: 0x003a, 0x1ae9: 0x002a, + 0x1aea: 0x000a, 0x1aeb: 0x000a, 0x1aec: 0x000a, 0x1aed: 0x000a, 0x1aee: 0x000a, 0x1aef: 0x000a, + 0x1af0: 0x000a, 0x1af1: 0x000a, 0x1af2: 0x000a, 0x1af3: 0x000a, 0x1af4: 0x000a, 0x1af5: 0x000a, + 0x1af6: 0x000a, 0x1af7: 0x000a, 0x1af8: 0x000a, 0x1af9: 0x000a, 0x1afa: 0x000a, 0x1afb: 0x000a, + 0x1afc: 0x000a, 0x1afd: 0x000a, 0x1afe: 0x000a, 0x1aff: 0x000a, + // Block 0x6c, offset 0x1b00 + 0x1b00: 0x000a, 0x1b01: 0x000a, 0x1b02: 0x000a, 0x1b03: 0x000a, 0x1b04: 0x000a, + // Block 0x6d, offset 0x1b40 + 0x1b40: 0x000a, 0x1b41: 0x000a, 0x1b42: 0x000a, 0x1b43: 0x000a, 0x1b44: 0x000a, 0x1b45: 0x000a, + 0x1b46: 0x000a, 0x1b47: 0x000a, 0x1b48: 0x000a, 0x1b49: 0x000a, 0x1b4a: 0x000a, 0x1b4b: 0x000a, + 0x1b4c: 0x000a, 0x1b4d: 0x000a, 0x1b4e: 0x000a, 0x1b4f: 0x000a, 0x1b50: 0x000a, 0x1b51: 0x000a, + 0x1b52: 0x000a, 0x1b53: 0x000a, 0x1b54: 0x000a, 0x1b55: 0x000a, 0x1b56: 0x000a, 0x1b57: 0x000a, + 0x1b58: 0x000a, 0x1b59: 0x000a, 0x1b5b: 0x000a, 0x1b5c: 0x000a, 0x1b5d: 0x000a, + 0x1b5e: 0x000a, 0x1b5f: 0x000a, 0x1b60: 0x000a, 0x1b61: 0x000a, 0x1b62: 0x000a, 0x1b63: 0x000a, + 0x1b64: 0x000a, 0x1b65: 0x000a, 0x1b66: 0x000a, 0x1b67: 0x000a, 0x1b68: 0x000a, 0x1b69: 0x000a, + 0x1b6a: 0x000a, 0x1b6b: 0x000a, 0x1b6c: 0x000a, 0x1b6d: 0x000a, 0x1b6e: 0x000a, 0x1b6f: 0x000a, + 0x1b70: 0x000a, 0x1b71: 0x000a, 0x1b72: 0x000a, 0x1b73: 0x000a, 0x1b74: 0x000a, 0x1b75: 0x000a, + 0x1b76: 0x000a, 0x1b77: 0x000a, 0x1b78: 0x000a, 0x1b79: 0x000a, 0x1b7a: 0x000a, 0x1b7b: 0x000a, + 0x1b7c: 0x000a, 0x1b7d: 0x000a, 0x1b7e: 0x000a, 0x1b7f: 0x000a, + // Block 0x6e, offset 0x1b80 + 0x1b80: 0x000a, 0x1b81: 0x000a, 0x1b82: 0x000a, 0x1b83: 0x000a, 0x1b84: 0x000a, 0x1b85: 0x000a, + 0x1b86: 0x000a, 0x1b87: 0x000a, 0x1b88: 0x000a, 0x1b89: 0x000a, 0x1b8a: 0x000a, 0x1b8b: 0x000a, + 0x1b8c: 0x000a, 0x1b8d: 0x000a, 0x1b8e: 0x000a, 0x1b8f: 0x000a, 0x1b90: 0x000a, 0x1b91: 0x000a, + 0x1b92: 0x000a, 0x1b93: 0x000a, 0x1b94: 0x000a, 0x1b95: 0x000a, 0x1b96: 0x000a, 0x1b97: 0x000a, + 0x1b98: 0x000a, 0x1b99: 0x000a, 0x1b9a: 0x000a, 0x1b9b: 0x000a, 0x1b9c: 0x000a, 0x1b9d: 0x000a, + 0x1b9e: 0x000a, 0x1b9f: 0x000a, 0x1ba0: 0x000a, 0x1ba1: 0x000a, 0x1ba2: 0x000a, 0x1ba3: 0x000a, + 0x1ba4: 0x000a, 0x1ba5: 0x000a, 0x1ba6: 0x000a, 0x1ba7: 0x000a, 0x1ba8: 0x000a, 0x1ba9: 0x000a, + 0x1baa: 0x000a, 0x1bab: 0x000a, 0x1bac: 0x000a, 0x1bad: 0x000a, 0x1bae: 0x000a, 0x1baf: 0x000a, + 0x1bb0: 0x000a, 0x1bb1: 0x000a, 0x1bb2: 0x000a, 0x1bb3: 0x000a, + // Block 0x6f, offset 0x1bc0 + 0x1bc0: 0x000a, 0x1bc1: 0x000a, 0x1bc2: 0x000a, 0x1bc3: 0x000a, 0x1bc4: 0x000a, 0x1bc5: 0x000a, + 0x1bc6: 0x000a, 0x1bc7: 0x000a, 0x1bc8: 0x000a, 0x1bc9: 0x000a, 0x1bca: 0x000a, 0x1bcb: 0x000a, + 0x1bcc: 0x000a, 0x1bcd: 0x000a, 0x1bce: 0x000a, 0x1bcf: 0x000a, 0x1bd0: 0x000a, 0x1bd1: 0x000a, + 0x1bd2: 0x000a, 0x1bd3: 0x000a, 0x1bd4: 0x000a, 0x1bd5: 0x000a, + 0x1bf0: 0x000a, 0x1bf1: 0x000a, 0x1bf2: 0x000a, 0x1bf3: 0x000a, 0x1bf4: 0x000a, 0x1bf5: 0x000a, + 0x1bf6: 0x000a, 0x1bf7: 0x000a, 0x1bf8: 0x000a, 0x1bf9: 0x000a, 0x1bfa: 0x000a, 0x1bfb: 0x000a, + // Block 0x70, offset 0x1c00 + 0x1c00: 0x0009, 0x1c01: 0x000a, 0x1c02: 0x000a, 0x1c03: 0x000a, 0x1c04: 0x000a, + 0x1c08: 0x003a, 0x1c09: 0x002a, 0x1c0a: 0x003a, 0x1c0b: 0x002a, + 0x1c0c: 0x003a, 0x1c0d: 0x002a, 0x1c0e: 0x003a, 0x1c0f: 0x002a, 0x1c10: 0x003a, 0x1c11: 0x002a, + 0x1c12: 0x000a, 0x1c13: 0x000a, 0x1c14: 0x003a, 0x1c15: 0x002a, 0x1c16: 0x003a, 0x1c17: 0x002a, + 0x1c18: 0x003a, 0x1c19: 0x002a, 0x1c1a: 0x003a, 0x1c1b: 0x002a, 0x1c1c: 0x000a, 0x1c1d: 0x000a, + 0x1c1e: 0x000a, 0x1c1f: 0x000a, 0x1c20: 0x000a, + 0x1c2a: 0x000c, 0x1c2b: 0x000c, 0x1c2c: 0x000c, 0x1c2d: 0x000c, + 0x1c30: 0x000a, + 0x1c36: 0x000a, 0x1c37: 0x000a, + 0x1c3d: 0x000a, 0x1c3e: 0x000a, 0x1c3f: 0x000a, + // Block 0x71, offset 0x1c40 + 0x1c59: 0x000c, 0x1c5a: 0x000c, 0x1c5b: 0x000a, 0x1c5c: 0x000a, + 0x1c60: 0x000a, + // Block 0x72, offset 0x1c80 + 0x1cbb: 0x000a, + // Block 0x73, offset 0x1cc0 + 0x1cc0: 0x000a, 0x1cc1: 0x000a, 0x1cc2: 0x000a, 0x1cc3: 0x000a, 0x1cc4: 0x000a, 0x1cc5: 0x000a, + 0x1cc6: 0x000a, 0x1cc7: 0x000a, 0x1cc8: 0x000a, 0x1cc9: 0x000a, 0x1cca: 0x000a, 0x1ccb: 0x000a, + 0x1ccc: 0x000a, 0x1ccd: 0x000a, 0x1cce: 0x000a, 0x1ccf: 0x000a, 0x1cd0: 0x000a, 0x1cd1: 0x000a, + 0x1cd2: 0x000a, 0x1cd3: 0x000a, 0x1cd4: 0x000a, 0x1cd5: 0x000a, 0x1cd6: 0x000a, 0x1cd7: 0x000a, + 0x1cd8: 0x000a, 0x1cd9: 0x000a, 0x1cda: 0x000a, 0x1cdb: 0x000a, 0x1cdc: 0x000a, 0x1cdd: 0x000a, + 0x1cde: 0x000a, 0x1cdf: 0x000a, 0x1ce0: 0x000a, 0x1ce1: 0x000a, 0x1ce2: 0x000a, 0x1ce3: 0x000a, + // Block 0x74, offset 0x1d00 + 0x1d1d: 0x000a, + 0x1d1e: 0x000a, + // Block 0x75, offset 0x1d40 + 0x1d50: 0x000a, 0x1d51: 0x000a, + 0x1d52: 0x000a, 0x1d53: 0x000a, 0x1d54: 0x000a, 0x1d55: 0x000a, 0x1d56: 0x000a, 0x1d57: 0x000a, + 0x1d58: 0x000a, 0x1d59: 0x000a, 0x1d5a: 0x000a, 0x1d5b: 0x000a, 0x1d5c: 0x000a, 0x1d5d: 0x000a, + 0x1d5e: 0x000a, 0x1d5f: 0x000a, + 0x1d7c: 0x000a, 0x1d7d: 0x000a, 0x1d7e: 0x000a, + // Block 0x76, offset 0x1d80 + 0x1db1: 0x000a, 0x1db2: 0x000a, 0x1db3: 0x000a, 0x1db4: 0x000a, 0x1db5: 0x000a, + 0x1db6: 0x000a, 0x1db7: 0x000a, 0x1db8: 0x000a, 0x1db9: 0x000a, 0x1dba: 0x000a, 0x1dbb: 0x000a, + 0x1dbc: 0x000a, 0x1dbd: 0x000a, 0x1dbe: 0x000a, 0x1dbf: 0x000a, + // Block 0x77, offset 0x1dc0 + 0x1dcc: 0x000a, 0x1dcd: 0x000a, 0x1dce: 0x000a, 0x1dcf: 0x000a, + // Block 0x78, offset 0x1e00 + 0x1e37: 0x000a, 0x1e38: 0x000a, 0x1e39: 0x000a, 0x1e3a: 0x000a, + // Block 0x79, offset 0x1e40 + 0x1e5e: 0x000a, 0x1e5f: 0x000a, + 0x1e7f: 0x000a, + // Block 0x7a, offset 0x1e80 + 0x1e90: 0x000a, 0x1e91: 0x000a, + 0x1e92: 0x000a, 0x1e93: 0x000a, 0x1e94: 0x000a, 0x1e95: 0x000a, 0x1e96: 0x000a, 0x1e97: 0x000a, + 0x1e98: 0x000a, 0x1e99: 0x000a, 0x1e9a: 0x000a, 0x1e9b: 0x000a, 0x1e9c: 0x000a, 0x1e9d: 0x000a, + 0x1e9e: 0x000a, 0x1e9f: 0x000a, 0x1ea0: 0x000a, 0x1ea1: 0x000a, 0x1ea2: 0x000a, 0x1ea3: 0x000a, + 0x1ea4: 0x000a, 0x1ea5: 0x000a, 0x1ea6: 0x000a, 0x1ea7: 0x000a, 0x1ea8: 0x000a, 0x1ea9: 0x000a, + 0x1eaa: 0x000a, 0x1eab: 0x000a, 0x1eac: 0x000a, 0x1ead: 0x000a, 0x1eae: 0x000a, 0x1eaf: 0x000a, + 0x1eb0: 0x000a, 0x1eb1: 0x000a, 0x1eb2: 0x000a, 0x1eb3: 0x000a, 0x1eb4: 0x000a, 0x1eb5: 0x000a, + 0x1eb6: 0x000a, 0x1eb7: 0x000a, 0x1eb8: 0x000a, 0x1eb9: 0x000a, 0x1eba: 0x000a, 0x1ebb: 0x000a, + 0x1ebc: 0x000a, 0x1ebd: 0x000a, 0x1ebe: 0x000a, 0x1ebf: 0x000a, + // Block 0x7b, offset 0x1ec0 + 0x1ec0: 0x000a, 0x1ec1: 0x000a, 0x1ec2: 0x000a, 0x1ec3: 0x000a, 0x1ec4: 0x000a, 0x1ec5: 0x000a, + 0x1ec6: 0x000a, + // Block 0x7c, offset 0x1f00 + 0x1f0d: 0x000a, 0x1f0e: 0x000a, 0x1f0f: 0x000a, + // Block 0x7d, offset 0x1f40 + 0x1f6f: 0x000c, + 0x1f70: 0x000c, 0x1f71: 0x000c, 0x1f72: 0x000c, 0x1f73: 0x000a, 0x1f74: 0x000c, 0x1f75: 0x000c, + 0x1f76: 0x000c, 0x1f77: 0x000c, 0x1f78: 0x000c, 0x1f79: 0x000c, 0x1f7a: 0x000c, 0x1f7b: 0x000c, + 0x1f7c: 0x000c, 0x1f7d: 0x000c, 0x1f7e: 0x000a, 0x1f7f: 0x000a, + // Block 0x7e, offset 0x1f80 + 0x1f9e: 0x000c, 0x1f9f: 0x000c, + // Block 0x7f, offset 0x1fc0 + 0x1ff0: 0x000c, 0x1ff1: 0x000c, + // Block 0x80, offset 0x2000 + 0x2000: 0x000a, 0x2001: 0x000a, 0x2002: 0x000a, 0x2003: 0x000a, 0x2004: 0x000a, 0x2005: 0x000a, + 0x2006: 0x000a, 0x2007: 0x000a, 0x2008: 0x000a, 0x2009: 0x000a, 0x200a: 0x000a, 0x200b: 0x000a, + 0x200c: 0x000a, 0x200d: 0x000a, 0x200e: 0x000a, 0x200f: 0x000a, 0x2010: 0x000a, 0x2011: 0x000a, + 0x2012: 0x000a, 0x2013: 0x000a, 0x2014: 0x000a, 0x2015: 0x000a, 0x2016: 0x000a, 0x2017: 0x000a, + 0x2018: 0x000a, 0x2019: 0x000a, 0x201a: 0x000a, 0x201b: 0x000a, 0x201c: 0x000a, 0x201d: 0x000a, + 0x201e: 0x000a, 0x201f: 0x000a, 0x2020: 0x000a, 0x2021: 0x000a, + // Block 0x81, offset 0x2040 + 0x2048: 0x000a, + // Block 0x82, offset 0x2080 + 0x2082: 0x000c, + 0x2086: 0x000c, 0x208b: 0x000c, + 0x20a5: 0x000c, 0x20a6: 0x000c, 0x20a8: 0x000a, 0x20a9: 0x000a, + 0x20aa: 0x000a, 0x20ab: 0x000a, + 0x20b8: 0x0004, 0x20b9: 0x0004, + // Block 0x83, offset 0x20c0 + 0x20f4: 0x000a, 0x20f5: 0x000a, + 0x20f6: 0x000a, 0x20f7: 0x000a, + // Block 0x84, offset 0x2100 + 0x2104: 0x000c, 0x2105: 0x000c, + 0x2120: 0x000c, 0x2121: 0x000c, 0x2122: 0x000c, 0x2123: 0x000c, + 0x2124: 0x000c, 0x2125: 0x000c, 0x2126: 0x000c, 0x2127: 0x000c, 0x2128: 0x000c, 0x2129: 0x000c, + 0x212a: 0x000c, 0x212b: 0x000c, 0x212c: 0x000c, 0x212d: 0x000c, 0x212e: 0x000c, 0x212f: 0x000c, + 0x2130: 0x000c, 0x2131: 0x000c, + // Block 0x85, offset 0x2140 + 0x2166: 0x000c, 0x2167: 0x000c, 0x2168: 0x000c, 0x2169: 0x000c, + 0x216a: 0x000c, 0x216b: 0x000c, 0x216c: 0x000c, 0x216d: 0x000c, + // Block 0x86, offset 0x2180 + 0x2187: 0x000c, 0x2188: 0x000c, 0x2189: 0x000c, 0x218a: 0x000c, 0x218b: 0x000c, + 0x218c: 0x000c, 0x218d: 0x000c, 0x218e: 0x000c, 0x218f: 0x000c, 0x2190: 0x000c, 0x2191: 0x000c, + // Block 0x87, offset 0x21c0 + 0x21c0: 0x000c, 0x21c1: 0x000c, 0x21c2: 0x000c, + 0x21f3: 0x000c, + 0x21f6: 0x000c, 0x21f7: 0x000c, 0x21f8: 0x000c, 0x21f9: 0x000c, + 0x21fc: 0x000c, + // Block 0x88, offset 0x2200 + 0x2225: 0x000c, + // Block 0x89, offset 0x2240 + 0x2269: 0x000c, + 0x226a: 0x000c, 0x226b: 0x000c, 0x226c: 0x000c, 0x226d: 0x000c, 0x226e: 0x000c, + 0x2271: 0x000c, 0x2272: 0x000c, 0x2275: 0x000c, + 0x2276: 0x000c, + // Block 0x8a, offset 0x2280 + 0x2283: 0x000c, + 0x228c: 0x000c, + 0x22bc: 0x000c, + // Block 0x8b, offset 0x22c0 + 0x22f0: 0x000c, 0x22f2: 0x000c, 0x22f3: 0x000c, 0x22f4: 0x000c, + 0x22f7: 0x000c, 0x22f8: 0x000c, + 0x22fe: 0x000c, 0x22ff: 0x000c, + // Block 0x8c, offset 0x2300 + 0x2301: 0x000c, + 0x232c: 0x000c, 0x232d: 0x000c, + 0x2336: 0x000c, + // Block 0x8d, offset 0x2340 + 0x2365: 0x000c, 0x2368: 0x000c, + 0x236d: 0x000c, + // Block 0x8e, offset 0x2380 + 0x239d: 0x0001, + 0x239e: 0x000c, 0x239f: 0x0001, 0x23a0: 0x0001, 0x23a1: 0x0001, 0x23a2: 0x0001, 0x23a3: 0x0001, + 0x23a4: 0x0001, 0x23a5: 0x0001, 0x23a6: 0x0001, 0x23a7: 0x0001, 0x23a8: 0x0001, 0x23a9: 0x0003, + 0x23aa: 0x0001, 0x23ab: 0x0001, 0x23ac: 0x0001, 0x23ad: 0x0001, 0x23ae: 0x0001, 0x23af: 0x0001, + 0x23b0: 0x0001, 0x23b1: 0x0001, 0x23b2: 0x0001, 0x23b3: 0x0001, 0x23b4: 0x0001, 0x23b5: 0x0001, + 0x23b6: 0x0001, 0x23b7: 0x0001, 0x23b8: 0x0001, 0x23b9: 0x0001, 0x23ba: 0x0001, 0x23bb: 0x0001, + 0x23bc: 0x0001, 0x23bd: 0x0001, 0x23be: 0x0001, 0x23bf: 0x0001, + // Block 0x8f, offset 0x23c0 + 0x23c0: 0x0001, 0x23c1: 0x0001, 0x23c2: 0x0001, 0x23c3: 0x0001, 0x23c4: 0x0001, 0x23c5: 0x0001, + 0x23c6: 0x0001, 0x23c7: 0x0001, 0x23c8: 0x0001, 0x23c9: 0x0001, 0x23ca: 0x0001, 0x23cb: 0x0001, + 0x23cc: 0x0001, 0x23cd: 0x0001, 0x23ce: 0x0001, 0x23cf: 0x0001, 0x23d0: 0x000d, 0x23d1: 0x000d, + 0x23d2: 0x000d, 0x23d3: 0x000d, 0x23d4: 0x000d, 0x23d5: 0x000d, 0x23d6: 0x000d, 0x23d7: 0x000d, + 0x23d8: 0x000d, 0x23d9: 0x000d, 0x23da: 0x000d, 0x23db: 0x000d, 0x23dc: 0x000d, 0x23dd: 0x000d, + 0x23de: 0x000d, 0x23df: 0x000d, 0x23e0: 0x000d, 0x23e1: 0x000d, 0x23e2: 0x000d, 0x23e3: 0x000d, + 0x23e4: 0x000d, 0x23e5: 0x000d, 0x23e6: 0x000d, 0x23e7: 0x000d, 0x23e8: 0x000d, 0x23e9: 0x000d, + 0x23ea: 0x000d, 0x23eb: 0x000d, 0x23ec: 0x000d, 0x23ed: 0x000d, 0x23ee: 0x000d, 0x23ef: 0x000d, + 0x23f0: 0x000d, 0x23f1: 0x000d, 0x23f2: 0x000d, 0x23f3: 0x000d, 0x23f4: 0x000d, 0x23f5: 0x000d, + 0x23f6: 0x000d, 0x23f7: 0x000d, 0x23f8: 0x000d, 0x23f9: 0x000d, 0x23fa: 0x000d, 0x23fb: 0x000d, + 0x23fc: 0x000d, 0x23fd: 0x000d, 0x23fe: 0x000d, 0x23ff: 0x000d, + // Block 0x90, offset 0x2400 + 0x2400: 0x000d, 0x2401: 0x000d, 0x2402: 0x000d, 0x2403: 0x000d, 0x2404: 0x000d, 0x2405: 0x000d, + 0x2406: 0x000d, 0x2407: 0x000d, 0x2408: 0x000d, 0x2409: 0x000d, 0x240a: 0x000d, 0x240b: 0x000d, + 0x240c: 0x000d, 0x240d: 0x000d, 0x240e: 0x000d, 0x240f: 0x000d, 0x2410: 0x000d, 0x2411: 0x000d, + 0x2412: 0x000d, 0x2413: 0x000d, 0x2414: 0x000d, 0x2415: 0x000d, 0x2416: 0x000d, 0x2417: 0x000d, + 0x2418: 0x000d, 0x2419: 0x000d, 0x241a: 0x000d, 0x241b: 0x000d, 0x241c: 0x000d, 0x241d: 0x000d, + 0x241e: 0x000d, 0x241f: 0x000d, 0x2420: 0x000d, 0x2421: 0x000d, 0x2422: 0x000d, 0x2423: 0x000d, + 0x2424: 0x000d, 0x2425: 0x000d, 0x2426: 0x000d, 0x2427: 0x000d, 0x2428: 0x000d, 0x2429: 0x000d, + 0x242a: 0x000d, 0x242b: 0x000d, 0x242c: 0x000d, 0x242d: 0x000d, 0x242e: 0x000d, 0x242f: 0x000d, + 0x2430: 0x000d, 0x2431: 0x000d, 0x2432: 0x000d, 0x2433: 0x000d, 0x2434: 0x000d, 0x2435: 0x000d, + 0x2436: 0x000d, 0x2437: 0x000d, 0x2438: 0x000d, 0x2439: 0x000d, 0x243a: 0x000d, 0x243b: 0x000d, + 0x243c: 0x000d, 0x243d: 0x000d, 0x243e: 0x000a, 0x243f: 0x000a, + // Block 0x91, offset 0x2440 + 0x2440: 0x000d, 0x2441: 0x000d, 0x2442: 0x000d, 0x2443: 0x000d, 0x2444: 0x000d, 0x2445: 0x000d, + 0x2446: 0x000d, 0x2447: 0x000d, 0x2448: 0x000d, 0x2449: 0x000d, 0x244a: 0x000d, 0x244b: 0x000d, + 0x244c: 0x000d, 0x244d: 0x000d, 0x244e: 0x000d, 0x244f: 0x000d, 0x2450: 0x000b, 0x2451: 0x000b, + 0x2452: 0x000b, 0x2453: 0x000b, 0x2454: 0x000b, 0x2455: 0x000b, 0x2456: 0x000b, 0x2457: 0x000b, + 0x2458: 0x000b, 0x2459: 0x000b, 0x245a: 0x000b, 0x245b: 0x000b, 0x245c: 0x000b, 0x245d: 0x000b, + 0x245e: 0x000b, 0x245f: 0x000b, 0x2460: 0x000b, 0x2461: 0x000b, 0x2462: 0x000b, 0x2463: 0x000b, + 0x2464: 0x000b, 0x2465: 0x000b, 0x2466: 0x000b, 0x2467: 0x000b, 0x2468: 0x000b, 0x2469: 0x000b, + 0x246a: 0x000b, 0x246b: 0x000b, 0x246c: 0x000b, 0x246d: 0x000b, 0x246e: 0x000b, 0x246f: 0x000b, + 0x2470: 0x000d, 0x2471: 0x000d, 0x2472: 0x000d, 0x2473: 0x000d, 0x2474: 0x000d, 0x2475: 0x000d, + 0x2476: 0x000d, 0x2477: 0x000d, 0x2478: 0x000d, 0x2479: 0x000d, 0x247a: 0x000d, 0x247b: 0x000d, + 0x247c: 0x000d, 0x247d: 0x000a, 0x247e: 0x000d, 0x247f: 0x000d, + // Block 0x92, offset 0x2480 + 0x2480: 0x000c, 0x2481: 0x000c, 0x2482: 0x000c, 0x2483: 0x000c, 0x2484: 0x000c, 0x2485: 0x000c, + 0x2486: 0x000c, 0x2487: 0x000c, 0x2488: 0x000c, 0x2489: 0x000c, 0x248a: 0x000c, 0x248b: 0x000c, + 0x248c: 0x000c, 0x248d: 0x000c, 0x248e: 0x000c, 0x248f: 0x000c, 0x2490: 0x000a, 0x2491: 0x000a, + 0x2492: 0x000a, 0x2493: 0x000a, 0x2494: 0x000a, 0x2495: 0x000a, 0x2496: 0x000a, 0x2497: 0x000a, + 0x2498: 0x000a, 0x2499: 0x000a, + 0x24a0: 0x000c, 0x24a1: 0x000c, 0x24a2: 0x000c, 0x24a3: 0x000c, + 0x24a4: 0x000c, 0x24a5: 0x000c, 0x24a6: 0x000c, 0x24a7: 0x000c, 0x24a8: 0x000c, 0x24a9: 0x000c, + 0x24aa: 0x000c, 0x24ab: 0x000c, 0x24ac: 0x000c, 0x24ad: 0x000c, 0x24ae: 0x000c, 0x24af: 0x000c, + 0x24b0: 0x000a, 0x24b1: 0x000a, 0x24b2: 0x000a, 0x24b3: 0x000a, 0x24b4: 0x000a, 0x24b5: 0x000a, + 0x24b6: 0x000a, 0x24b7: 0x000a, 0x24b8: 0x000a, 0x24b9: 0x000a, 0x24ba: 0x000a, 0x24bb: 0x000a, + 0x24bc: 0x000a, 0x24bd: 0x000a, 0x24be: 0x000a, 0x24bf: 0x000a, + // Block 0x93, offset 0x24c0 + 0x24c0: 0x000a, 0x24c1: 0x000a, 0x24c2: 0x000a, 0x24c3: 0x000a, 0x24c4: 0x000a, 0x24c5: 0x000a, + 0x24c6: 0x000a, 0x24c7: 0x000a, 0x24c8: 0x000a, 0x24c9: 0x000a, 0x24ca: 0x000a, 0x24cb: 0x000a, + 0x24cc: 0x000a, 0x24cd: 0x000a, 0x24ce: 0x000a, 0x24cf: 0x000a, 0x24d0: 0x0006, 0x24d1: 0x000a, + 0x24d2: 0x0006, 0x24d4: 0x000a, 0x24d5: 0x0006, 0x24d6: 0x000a, 0x24d7: 0x000a, + 0x24d8: 0x000a, 0x24d9: 0x009a, 0x24da: 0x008a, 0x24db: 0x007a, 0x24dc: 0x006a, 0x24dd: 0x009a, + 0x24de: 0x008a, 0x24df: 0x0004, 0x24e0: 0x000a, 0x24e1: 0x000a, 0x24e2: 0x0003, 0x24e3: 0x0003, + 0x24e4: 0x000a, 0x24e5: 0x000a, 0x24e6: 0x000a, 0x24e8: 0x000a, 0x24e9: 0x0004, + 0x24ea: 0x0004, 0x24eb: 0x000a, + 0x24f0: 0x000d, 0x24f1: 0x000d, 0x24f2: 0x000d, 0x24f3: 0x000d, 0x24f4: 0x000d, 0x24f5: 0x000d, + 0x24f6: 0x000d, 0x24f7: 0x000d, 0x24f8: 0x000d, 0x24f9: 0x000d, 0x24fa: 0x000d, 0x24fb: 0x000d, + 0x24fc: 0x000d, 0x24fd: 0x000d, 0x24fe: 0x000d, 0x24ff: 0x000d, + // Block 0x94, offset 0x2500 + 0x2500: 0x000d, 0x2501: 0x000d, 0x2502: 0x000d, 0x2503: 0x000d, 0x2504: 0x000d, 0x2505: 0x000d, + 0x2506: 0x000d, 0x2507: 0x000d, 0x2508: 0x000d, 0x2509: 0x000d, 0x250a: 0x000d, 0x250b: 0x000d, + 0x250c: 0x000d, 0x250d: 0x000d, 0x250e: 0x000d, 0x250f: 0x000d, 0x2510: 0x000d, 0x2511: 0x000d, + 0x2512: 0x000d, 0x2513: 0x000d, 0x2514: 0x000d, 0x2515: 0x000d, 0x2516: 0x000d, 0x2517: 0x000d, + 0x2518: 0x000d, 0x2519: 0x000d, 0x251a: 0x000d, 0x251b: 0x000d, 0x251c: 0x000d, 0x251d: 0x000d, + 0x251e: 0x000d, 0x251f: 0x000d, 0x2520: 0x000d, 0x2521: 0x000d, 0x2522: 0x000d, 0x2523: 0x000d, + 0x2524: 0x000d, 0x2525: 0x000d, 0x2526: 0x000d, 0x2527: 0x000d, 0x2528: 0x000d, 0x2529: 0x000d, + 0x252a: 0x000d, 0x252b: 0x000d, 0x252c: 0x000d, 0x252d: 0x000d, 0x252e: 0x000d, 0x252f: 0x000d, + 0x2530: 0x000d, 0x2531: 0x000d, 0x2532: 0x000d, 0x2533: 0x000d, 0x2534: 0x000d, 0x2535: 0x000d, + 0x2536: 0x000d, 0x2537: 0x000d, 0x2538: 0x000d, 0x2539: 0x000d, 0x253a: 0x000d, 0x253b: 0x000d, + 0x253c: 0x000d, 0x253d: 0x000d, 0x253e: 0x000d, 0x253f: 0x000b, + // Block 0x95, offset 0x2540 + 0x2541: 0x000a, 0x2542: 0x000a, 0x2543: 0x0004, 0x2544: 0x0004, 0x2545: 0x0004, + 0x2546: 0x000a, 0x2547: 0x000a, 0x2548: 0x003a, 0x2549: 0x002a, 0x254a: 0x000a, 0x254b: 0x0003, + 0x254c: 0x0006, 0x254d: 0x0003, 0x254e: 0x0006, 0x254f: 0x0006, 0x2550: 0x0002, 0x2551: 0x0002, + 0x2552: 0x0002, 0x2553: 0x0002, 0x2554: 0x0002, 0x2555: 0x0002, 0x2556: 0x0002, 0x2557: 0x0002, + 0x2558: 0x0002, 0x2559: 0x0002, 0x255a: 0x0006, 0x255b: 0x000a, 0x255c: 0x000a, 0x255d: 0x000a, + 0x255e: 0x000a, 0x255f: 0x000a, 0x2560: 0x000a, + 0x257b: 0x005a, + 0x257c: 0x000a, 0x257d: 0x004a, 0x257e: 0x000a, 0x257f: 0x000a, + // Block 0x96, offset 0x2580 + 0x2580: 0x000a, + 0x259b: 0x005a, 0x259c: 0x000a, 0x259d: 0x004a, + 0x259e: 0x000a, 0x259f: 0x00fa, 0x25a0: 0x00ea, 0x25a1: 0x000a, 0x25a2: 0x003a, 0x25a3: 0x002a, + 0x25a4: 0x000a, 0x25a5: 0x000a, + // Block 0x97, offset 0x25c0 + 0x25e0: 0x0004, 0x25e1: 0x0004, 0x25e2: 0x000a, 0x25e3: 0x000a, + 0x25e4: 0x000a, 0x25e5: 0x0004, 0x25e6: 0x0004, 0x25e8: 0x000a, 0x25e9: 0x000a, + 0x25ea: 0x000a, 0x25eb: 0x000a, 0x25ec: 0x000a, 0x25ed: 0x000a, 0x25ee: 0x000a, + 0x25f0: 0x000b, 0x25f1: 0x000b, 0x25f2: 0x000b, 0x25f3: 0x000b, 0x25f4: 0x000b, 0x25f5: 0x000b, + 0x25f6: 0x000b, 0x25f7: 0x000b, 0x25f8: 0x000b, 0x25f9: 0x000a, 0x25fa: 0x000a, 0x25fb: 0x000a, + 0x25fc: 0x000a, 0x25fd: 0x000a, 0x25fe: 0x000b, 0x25ff: 0x000b, + // Block 0x98, offset 0x2600 + 0x2601: 0x000a, + // Block 0x99, offset 0x2640 + 0x2640: 0x000a, 0x2641: 0x000a, 0x2642: 0x000a, 0x2643: 0x000a, 0x2644: 0x000a, 0x2645: 0x000a, + 0x2646: 0x000a, 0x2647: 0x000a, 0x2648: 0x000a, 0x2649: 0x000a, 0x264a: 0x000a, 0x264b: 0x000a, + 0x264c: 0x000a, 0x2650: 0x000a, 0x2651: 0x000a, + 0x2652: 0x000a, 0x2653: 0x000a, 0x2654: 0x000a, 0x2655: 0x000a, 0x2656: 0x000a, 0x2657: 0x000a, + 0x2658: 0x000a, 0x2659: 0x000a, 0x265a: 0x000a, 0x265b: 0x000a, + 0x2660: 0x000a, + // Block 0x9a, offset 0x2680 + 0x26bd: 0x000c, + // Block 0x9b, offset 0x26c0 + 0x26e0: 0x000c, 0x26e1: 0x0002, 0x26e2: 0x0002, 0x26e3: 0x0002, + 0x26e4: 0x0002, 0x26e5: 0x0002, 0x26e6: 0x0002, 0x26e7: 0x0002, 0x26e8: 0x0002, 0x26e9: 0x0002, + 0x26ea: 0x0002, 0x26eb: 0x0002, 0x26ec: 0x0002, 0x26ed: 0x0002, 0x26ee: 0x0002, 0x26ef: 0x0002, + 0x26f0: 0x0002, 0x26f1: 0x0002, 0x26f2: 0x0002, 0x26f3: 0x0002, 0x26f4: 0x0002, 0x26f5: 0x0002, + 0x26f6: 0x0002, 0x26f7: 0x0002, 0x26f8: 0x0002, 0x26f9: 0x0002, 0x26fa: 0x0002, 0x26fb: 0x0002, + // Block 0x9c, offset 0x2700 + 0x2736: 0x000c, 0x2737: 0x000c, 0x2738: 0x000c, 0x2739: 0x000c, 0x273a: 0x000c, + // Block 0x9d, offset 0x2740 + 0x2740: 0x0001, 0x2741: 0x0001, 0x2742: 0x0001, 0x2743: 0x0001, 0x2744: 0x0001, 0x2745: 0x0001, + 0x2746: 0x0001, 0x2747: 0x0001, 0x2748: 0x0001, 0x2749: 0x0001, 0x274a: 0x0001, 0x274b: 0x0001, + 0x274c: 0x0001, 0x274d: 0x0001, 0x274e: 0x0001, 0x274f: 0x0001, 0x2750: 0x0001, 0x2751: 0x0001, + 0x2752: 0x0001, 0x2753: 0x0001, 0x2754: 0x0001, 0x2755: 0x0001, 0x2756: 0x0001, 0x2757: 0x0001, + 0x2758: 0x0001, 0x2759: 0x0001, 0x275a: 0x0001, 0x275b: 0x0001, 0x275c: 0x0001, 0x275d: 0x0001, + 0x275e: 0x0001, 0x275f: 0x0001, 0x2760: 0x0001, 0x2761: 0x0001, 0x2762: 0x0001, 0x2763: 0x0001, + 0x2764: 0x0001, 0x2765: 0x0001, 0x2766: 0x0001, 0x2767: 0x0001, 0x2768: 0x0001, 0x2769: 0x0001, + 0x276a: 0x0001, 0x276b: 0x0001, 0x276c: 0x0001, 0x276d: 0x0001, 0x276e: 0x0001, 0x276f: 0x0001, + 0x2770: 0x0001, 0x2771: 0x0001, 0x2772: 0x0001, 0x2773: 0x0001, 0x2774: 0x0001, 0x2775: 0x0001, + 0x2776: 0x0001, 0x2777: 0x0001, 0x2778: 0x0001, 0x2779: 0x0001, 0x277a: 0x0001, 0x277b: 0x0001, + 0x277c: 0x0001, 0x277d: 0x0001, 0x277e: 0x0001, 0x277f: 0x0001, + // Block 0x9e, offset 0x2780 + 0x2780: 0x0001, 0x2781: 0x0001, 0x2782: 0x0001, 0x2783: 0x0001, 0x2784: 0x0001, 0x2785: 0x0001, + 0x2786: 0x0001, 0x2787: 0x0001, 0x2788: 0x0001, 0x2789: 0x0001, 0x278a: 0x0001, 0x278b: 0x0001, + 0x278c: 0x0001, 0x278d: 0x0001, 0x278e: 0x0001, 0x278f: 0x0001, 0x2790: 0x0001, 0x2791: 0x0001, + 0x2792: 0x0001, 0x2793: 0x0001, 0x2794: 0x0001, 0x2795: 0x0001, 0x2796: 0x0001, 0x2797: 0x0001, + 0x2798: 0x0001, 0x2799: 0x0001, 0x279a: 0x0001, 0x279b: 0x0001, 0x279c: 0x0001, 0x279d: 0x0001, + 0x279e: 0x0001, 0x279f: 0x000a, 0x27a0: 0x0001, 0x27a1: 0x0001, 0x27a2: 0x0001, 0x27a3: 0x0001, + 0x27a4: 0x0001, 0x27a5: 0x0001, 0x27a6: 0x0001, 0x27a7: 0x0001, 0x27a8: 0x0001, 0x27a9: 0x0001, + 0x27aa: 0x0001, 0x27ab: 0x0001, 0x27ac: 0x0001, 0x27ad: 0x0001, 0x27ae: 0x0001, 0x27af: 0x0001, + 0x27b0: 0x0001, 0x27b1: 0x0001, 0x27b2: 0x0001, 0x27b3: 0x0001, 0x27b4: 0x0001, 0x27b5: 0x0001, + 0x27b6: 0x0001, 0x27b7: 0x0001, 0x27b8: 0x0001, 0x27b9: 0x0001, 0x27ba: 0x0001, 0x27bb: 0x0001, + 0x27bc: 0x0001, 0x27bd: 0x0001, 0x27be: 0x0001, 0x27bf: 0x0001, + // Block 0x9f, offset 0x27c0 + 0x27c0: 0x0001, 0x27c1: 0x000c, 0x27c2: 0x000c, 0x27c3: 0x000c, 0x27c4: 0x0001, 0x27c5: 0x000c, + 0x27c6: 0x000c, 0x27c7: 0x0001, 0x27c8: 0x0001, 0x27c9: 0x0001, 0x27ca: 0x0001, 0x27cb: 0x0001, + 0x27cc: 0x000c, 0x27cd: 0x000c, 0x27ce: 0x000c, 0x27cf: 0x000c, 0x27d0: 0x0001, 0x27d1: 0x0001, + 0x27d2: 0x0001, 0x27d3: 0x0001, 0x27d4: 0x0001, 0x27d5: 0x0001, 0x27d6: 0x0001, 0x27d7: 0x0001, + 0x27d8: 0x0001, 0x27d9: 0x0001, 0x27da: 0x0001, 0x27db: 0x0001, 0x27dc: 0x0001, 0x27dd: 0x0001, + 0x27de: 0x0001, 0x27df: 0x0001, 0x27e0: 0x0001, 0x27e1: 0x0001, 0x27e2: 0x0001, 0x27e3: 0x0001, + 0x27e4: 0x0001, 0x27e5: 0x0001, 0x27e6: 0x0001, 0x27e7: 0x0001, 0x27e8: 0x0001, 0x27e9: 0x0001, + 0x27ea: 0x0001, 0x27eb: 0x0001, 0x27ec: 0x0001, 0x27ed: 0x0001, 0x27ee: 0x0001, 0x27ef: 0x0001, + 0x27f0: 0x0001, 0x27f1: 0x0001, 0x27f2: 0x0001, 0x27f3: 0x0001, 0x27f4: 0x0001, 0x27f5: 0x0001, + 0x27f6: 0x0001, 0x27f7: 0x0001, 0x27f8: 0x000c, 0x27f9: 0x000c, 0x27fa: 0x000c, 0x27fb: 0x0001, + 0x27fc: 0x0001, 0x27fd: 0x0001, 0x27fe: 0x0001, 0x27ff: 0x000c, + // Block 0xa0, offset 0x2800 + 0x2800: 0x0001, 0x2801: 0x0001, 0x2802: 0x0001, 0x2803: 0x0001, 0x2804: 0x0001, 0x2805: 0x0001, + 0x2806: 0x0001, 0x2807: 0x0001, 0x2808: 0x0001, 0x2809: 0x0001, 0x280a: 0x0001, 0x280b: 0x0001, + 0x280c: 0x0001, 0x280d: 0x0001, 0x280e: 0x0001, 0x280f: 0x0001, 0x2810: 0x0001, 0x2811: 0x0001, + 0x2812: 0x0001, 0x2813: 0x0001, 0x2814: 0x0001, 0x2815: 0x0001, 0x2816: 0x0001, 0x2817: 0x0001, + 0x2818: 0x0001, 0x2819: 0x0001, 0x281a: 0x0001, 0x281b: 0x0001, 0x281c: 0x0001, 0x281d: 0x0001, + 0x281e: 0x0001, 0x281f: 0x0001, 0x2820: 0x0001, 0x2821: 0x0001, 0x2822: 0x0001, 0x2823: 0x0001, + 0x2824: 0x0001, 0x2825: 0x000c, 0x2826: 0x000c, 0x2827: 0x0001, 0x2828: 0x0001, 0x2829: 0x0001, + 0x282a: 0x0001, 0x282b: 0x0001, 0x282c: 0x0001, 0x282d: 0x0001, 0x282e: 0x0001, 0x282f: 0x0001, + 0x2830: 0x0001, 0x2831: 0x0001, 0x2832: 0x0001, 0x2833: 0x0001, 0x2834: 0x0001, 0x2835: 0x0001, + 0x2836: 0x0001, 0x2837: 0x0001, 0x2838: 0x0001, 0x2839: 0x0001, 0x283a: 0x0001, 0x283b: 0x0001, + 0x283c: 0x0001, 0x283d: 0x0001, 0x283e: 0x0001, 0x283f: 0x0001, + // Block 0xa1, offset 0x2840 + 0x2840: 0x0001, 0x2841: 0x0001, 0x2842: 0x0001, 0x2843: 0x0001, 0x2844: 0x0001, 0x2845: 0x0001, + 0x2846: 0x0001, 0x2847: 0x0001, 0x2848: 0x0001, 0x2849: 0x0001, 0x284a: 0x0001, 0x284b: 0x0001, + 0x284c: 0x0001, 0x284d: 0x0001, 0x284e: 0x0001, 0x284f: 0x0001, 0x2850: 0x0001, 0x2851: 0x0001, + 0x2852: 0x0001, 0x2853: 0x0001, 0x2854: 0x0001, 0x2855: 0x0001, 0x2856: 0x0001, 0x2857: 0x0001, + 0x2858: 0x0001, 0x2859: 0x0001, 0x285a: 0x0001, 0x285b: 0x0001, 0x285c: 0x0001, 0x285d: 0x0001, + 0x285e: 0x0001, 0x285f: 0x0001, 0x2860: 0x0001, 0x2861: 0x0001, 0x2862: 0x0001, 0x2863: 0x0001, + 0x2864: 0x0001, 0x2865: 0x0001, 0x2866: 0x0001, 0x2867: 0x0001, 0x2868: 0x0001, 0x2869: 0x0001, + 0x286a: 0x0001, 0x286b: 0x0001, 0x286c: 0x0001, 0x286d: 0x0001, 0x286e: 0x0001, 0x286f: 0x0001, + 0x2870: 0x0001, 0x2871: 0x0001, 0x2872: 0x0001, 0x2873: 0x0001, 0x2874: 0x0001, 0x2875: 0x0001, + 0x2876: 0x0001, 0x2877: 0x0001, 0x2878: 0x0001, 0x2879: 0x000a, 0x287a: 0x000a, 0x287b: 0x000a, + 0x287c: 0x000a, 0x287d: 0x000a, 0x287e: 0x000a, 0x287f: 0x000a, + // Block 0xa2, offset 0x2880 + 0x2880: 0x0001, 0x2881: 0x0001, 0x2882: 0x0001, 0x2883: 0x0001, 0x2884: 0x0001, 0x2885: 0x0001, + 0x2886: 0x0001, 0x2887: 0x0001, 0x2888: 0x0001, 0x2889: 0x0001, 0x288a: 0x0001, 0x288b: 0x0001, + 0x288c: 0x0001, 0x288d: 0x0001, 0x288e: 0x0001, 0x288f: 0x0001, 0x2890: 0x0001, 0x2891: 0x0001, + 0x2892: 0x0001, 0x2893: 0x0001, 0x2894: 0x0001, 0x2895: 0x0001, 0x2896: 0x0001, 0x2897: 0x0001, + 0x2898: 0x0001, 0x2899: 0x0001, 0x289a: 0x0001, 0x289b: 0x0001, 0x289c: 0x0001, 0x289d: 0x0001, + 0x289e: 0x0001, 0x289f: 0x0001, 0x28a0: 0x0005, 0x28a1: 0x0005, 0x28a2: 0x0005, 0x28a3: 0x0005, + 0x28a4: 0x0005, 0x28a5: 0x0005, 0x28a6: 0x0005, 0x28a7: 0x0005, 0x28a8: 0x0005, 0x28a9: 0x0005, + 0x28aa: 0x0005, 0x28ab: 0x0005, 0x28ac: 0x0005, 0x28ad: 0x0005, 0x28ae: 0x0005, 0x28af: 0x0005, + 0x28b0: 0x0005, 0x28b1: 0x0005, 0x28b2: 0x0005, 0x28b3: 0x0005, 0x28b4: 0x0005, 0x28b5: 0x0005, + 0x28b6: 0x0005, 0x28b7: 0x0005, 0x28b8: 0x0005, 0x28b9: 0x0005, 0x28ba: 0x0005, 0x28bb: 0x0005, + 0x28bc: 0x0005, 0x28bd: 0x0005, 0x28be: 0x0005, 0x28bf: 0x0001, + // Block 0xa3, offset 0x28c0 + 0x28c1: 0x000c, + 0x28f8: 0x000c, 0x28f9: 0x000c, 0x28fa: 0x000c, 0x28fb: 0x000c, + 0x28fc: 0x000c, 0x28fd: 0x000c, 0x28fe: 0x000c, 0x28ff: 0x000c, + // Block 0xa4, offset 0x2900 + 0x2900: 0x000c, 0x2901: 0x000c, 0x2902: 0x000c, 0x2903: 0x000c, 0x2904: 0x000c, 0x2905: 0x000c, + 0x2906: 0x000c, + 0x2912: 0x000a, 0x2913: 0x000a, 0x2914: 0x000a, 0x2915: 0x000a, 0x2916: 0x000a, 0x2917: 0x000a, + 0x2918: 0x000a, 0x2919: 0x000a, 0x291a: 0x000a, 0x291b: 0x000a, 0x291c: 0x000a, 0x291d: 0x000a, + 0x291e: 0x000a, 0x291f: 0x000a, 0x2920: 0x000a, 0x2921: 0x000a, 0x2922: 0x000a, 0x2923: 0x000a, + 0x2924: 0x000a, 0x2925: 0x000a, + 0x293f: 0x000c, + // Block 0xa5, offset 0x2940 + 0x2940: 0x000c, 0x2941: 0x000c, + 0x2973: 0x000c, 0x2974: 0x000c, 0x2975: 0x000c, + 0x2976: 0x000c, 0x2979: 0x000c, 0x297a: 0x000c, + // Block 0xa6, offset 0x2980 + 0x2980: 0x000c, 0x2981: 0x000c, 0x2982: 0x000c, + 0x29a7: 0x000c, 0x29a8: 0x000c, 0x29a9: 0x000c, + 0x29aa: 0x000c, 0x29ab: 0x000c, 0x29ad: 0x000c, 0x29ae: 0x000c, 0x29af: 0x000c, + 0x29b0: 0x000c, 0x29b1: 0x000c, 0x29b2: 0x000c, 0x29b3: 0x000c, 0x29b4: 0x000c, + // Block 0xa7, offset 0x29c0 + 0x29f3: 0x000c, + // Block 0xa8, offset 0x2a00 + 0x2a00: 0x000c, 0x2a01: 0x000c, + 0x2a36: 0x000c, 0x2a37: 0x000c, 0x2a38: 0x000c, 0x2a39: 0x000c, 0x2a3a: 0x000c, 0x2a3b: 0x000c, + 0x2a3c: 0x000c, 0x2a3d: 0x000c, 0x2a3e: 0x000c, + // Block 0xa9, offset 0x2a40 + 0x2a4a: 0x000c, 0x2a4b: 0x000c, + 0x2a4c: 0x000c, + // Block 0xaa, offset 0x2a80 + 0x2aaf: 0x000c, + 0x2ab0: 0x000c, 0x2ab1: 0x000c, 0x2ab4: 0x000c, + 0x2ab6: 0x000c, 0x2ab7: 0x000c, + 0x2abe: 0x000c, + // Block 0xab, offset 0x2ac0 + 0x2adf: 0x000c, 0x2ae3: 0x000c, + 0x2ae4: 0x000c, 0x2ae5: 0x000c, 0x2ae6: 0x000c, 0x2ae7: 0x000c, 0x2ae8: 0x000c, 0x2ae9: 0x000c, + 0x2aea: 0x000c, + // Block 0xac, offset 0x2b00 + 0x2b00: 0x000c, 0x2b01: 0x000c, + 0x2b3c: 0x000c, + // Block 0xad, offset 0x2b40 + 0x2b40: 0x000c, + 0x2b66: 0x000c, 0x2b67: 0x000c, 0x2b68: 0x000c, 0x2b69: 0x000c, + 0x2b6a: 0x000c, 0x2b6b: 0x000c, 0x2b6c: 0x000c, + 0x2b70: 0x000c, 0x2b71: 0x000c, 0x2b72: 0x000c, 0x2b73: 0x000c, 0x2b74: 0x000c, + // Block 0xae, offset 0x2b80 + 0x2bb8: 0x000c, 0x2bb9: 0x000c, 0x2bba: 0x000c, 0x2bbb: 0x000c, + 0x2bbc: 0x000c, 0x2bbd: 0x000c, 0x2bbe: 0x000c, 0x2bbf: 0x000c, + // Block 0xaf, offset 0x2bc0 + 0x2bc2: 0x000c, 0x2bc3: 0x000c, 0x2bc4: 0x000c, + 0x2bc6: 0x000c, + // Block 0xb0, offset 0x2c00 + 0x2c33: 0x000c, 0x2c34: 0x000c, 0x2c35: 0x000c, + 0x2c36: 0x000c, 0x2c37: 0x000c, 0x2c38: 0x000c, 0x2c3a: 0x000c, + 0x2c3f: 0x000c, + // Block 0xb1, offset 0x2c40 + 0x2c40: 0x000c, 0x2c42: 0x000c, 0x2c43: 0x000c, + // Block 0xb2, offset 0x2c80 + 0x2cb2: 0x000c, 0x2cb3: 0x000c, 0x2cb4: 0x000c, 0x2cb5: 0x000c, + 0x2cbc: 0x000c, 0x2cbd: 0x000c, 0x2cbf: 0x000c, + // Block 0xb3, offset 0x2cc0 + 0x2cc0: 0x000c, + 0x2cdc: 0x000c, 0x2cdd: 0x000c, + // Block 0xb4, offset 0x2d00 + 0x2d33: 0x000c, 0x2d34: 0x000c, 0x2d35: 0x000c, + 0x2d36: 0x000c, 0x2d37: 0x000c, 0x2d38: 0x000c, 0x2d39: 0x000c, 0x2d3a: 0x000c, + 0x2d3d: 0x000c, 0x2d3f: 0x000c, + // Block 0xb5, offset 0x2d40 + 0x2d40: 0x000c, + 0x2d60: 0x000a, 0x2d61: 0x000a, 0x2d62: 0x000a, 0x2d63: 0x000a, + 0x2d64: 0x000a, 0x2d65: 0x000a, 0x2d66: 0x000a, 0x2d67: 0x000a, 0x2d68: 0x000a, 0x2d69: 0x000a, + 0x2d6a: 0x000a, 0x2d6b: 0x000a, 0x2d6c: 0x000a, + // Block 0xb6, offset 0x2d80 + 0x2dab: 0x000c, 0x2dad: 0x000c, + 0x2db0: 0x000c, 0x2db1: 0x000c, 0x2db2: 0x000c, 0x2db3: 0x000c, 0x2db4: 0x000c, 0x2db5: 0x000c, + 0x2db7: 0x000c, + // Block 0xb7, offset 0x2dc0 + 0x2ddd: 0x000c, + 0x2dde: 0x000c, 0x2ddf: 0x000c, 0x2de2: 0x000c, 0x2de3: 0x000c, + 0x2de4: 0x000c, 0x2de5: 0x000c, 0x2de7: 0x000c, 0x2de8: 0x000c, 0x2de9: 0x000c, + 0x2dea: 0x000c, 0x2deb: 0x000c, + // Block 0xb8, offset 0x2e00 + 0x2e30: 0x000c, 0x2e31: 0x000c, 0x2e32: 0x000c, 0x2e33: 0x000c, 0x2e34: 0x000c, 0x2e35: 0x000c, + 0x2e36: 0x000c, 0x2e38: 0x000c, 0x2e39: 0x000c, 0x2e3a: 0x000c, 0x2e3b: 0x000c, + 0x2e3c: 0x000c, 0x2e3d: 0x000c, + // Block 0xb9, offset 0x2e40 + 0x2e52: 0x000c, 0x2e53: 0x000c, 0x2e54: 0x000c, 0x2e55: 0x000c, 0x2e56: 0x000c, 0x2e57: 0x000c, + 0x2e58: 0x000c, 0x2e59: 0x000c, 0x2e5a: 0x000c, 0x2e5b: 0x000c, 0x2e5c: 0x000c, 0x2e5d: 0x000c, + 0x2e5e: 0x000c, 0x2e5f: 0x000c, 0x2e60: 0x000c, 0x2e61: 0x000c, 0x2e62: 0x000c, 0x2e63: 0x000c, + 0x2e64: 0x000c, 0x2e65: 0x000c, 0x2e66: 0x000c, 0x2e67: 0x000c, + 0x2e6a: 0x000c, 0x2e6b: 0x000c, 0x2e6c: 0x000c, 0x2e6d: 0x000c, 0x2e6e: 0x000c, 0x2e6f: 0x000c, + 0x2e70: 0x000c, 0x2e72: 0x000c, 0x2e73: 0x000c, 0x2e75: 0x000c, + 0x2e76: 0x000c, + // Block 0xba, offset 0x2e80 + 0x2eb0: 0x000c, 0x2eb1: 0x000c, 0x2eb2: 0x000c, 0x2eb3: 0x000c, 0x2eb4: 0x000c, + // Block 0xbb, offset 0x2ec0 + 0x2ef0: 0x000c, 0x2ef1: 0x000c, 0x2ef2: 0x000c, 0x2ef3: 0x000c, 0x2ef4: 0x000c, 0x2ef5: 0x000c, + 0x2ef6: 0x000c, + // Block 0xbc, offset 0x2f00 + 0x2f0f: 0x000c, 0x2f10: 0x000c, 0x2f11: 0x000c, + 0x2f12: 0x000c, + // Block 0xbd, offset 0x2f40 + 0x2f5d: 0x000c, + 0x2f5e: 0x000c, 0x2f60: 0x000b, 0x2f61: 0x000b, 0x2f62: 0x000b, 0x2f63: 0x000b, + // Block 0xbe, offset 0x2f80 + 0x2fa7: 0x000c, 0x2fa8: 0x000c, 0x2fa9: 0x000c, + 0x2fb3: 0x000b, 0x2fb4: 0x000b, 0x2fb5: 0x000b, + 0x2fb6: 0x000b, 0x2fb7: 0x000b, 0x2fb8: 0x000b, 0x2fb9: 0x000b, 0x2fba: 0x000b, 0x2fbb: 0x000c, + 0x2fbc: 0x000c, 0x2fbd: 0x000c, 0x2fbe: 0x000c, 0x2fbf: 0x000c, + // Block 0xbf, offset 0x2fc0 + 0x2fc0: 0x000c, 0x2fc1: 0x000c, 0x2fc2: 0x000c, 0x2fc5: 0x000c, + 0x2fc6: 0x000c, 0x2fc7: 0x000c, 0x2fc8: 0x000c, 0x2fc9: 0x000c, 0x2fca: 0x000c, 0x2fcb: 0x000c, + 0x2fea: 0x000c, 0x2feb: 0x000c, 0x2fec: 0x000c, 0x2fed: 0x000c, + // Block 0xc0, offset 0x3000 + 0x3000: 0x000a, 0x3001: 0x000a, 0x3002: 0x000c, 0x3003: 0x000c, 0x3004: 0x000c, 0x3005: 0x000a, + // Block 0xc1, offset 0x3040 + 0x3040: 0x000a, 0x3041: 0x000a, 0x3042: 0x000a, 0x3043: 0x000a, 0x3044: 0x000a, 0x3045: 0x000a, + 0x3046: 0x000a, 0x3047: 0x000a, 0x3048: 0x000a, 0x3049: 0x000a, 0x304a: 0x000a, 0x304b: 0x000a, + 0x304c: 0x000a, 0x304d: 0x000a, 0x304e: 0x000a, 0x304f: 0x000a, 0x3050: 0x000a, 0x3051: 0x000a, + 0x3052: 0x000a, 0x3053: 0x000a, 0x3054: 0x000a, 0x3055: 0x000a, 0x3056: 0x000a, + // Block 0xc2, offset 0x3080 + 0x309b: 0x000a, + // Block 0xc3, offset 0x30c0 + 0x30d5: 0x000a, + // Block 0xc4, offset 0x3100 + 0x310f: 0x000a, + // Block 0xc5, offset 0x3140 + 0x3149: 0x000a, + // Block 0xc6, offset 0x3180 + 0x3183: 0x000a, + 0x318e: 0x0002, 0x318f: 0x0002, 0x3190: 0x0002, 0x3191: 0x0002, + 0x3192: 0x0002, 0x3193: 0x0002, 0x3194: 0x0002, 0x3195: 0x0002, 0x3196: 0x0002, 0x3197: 0x0002, + 0x3198: 0x0002, 0x3199: 0x0002, 0x319a: 0x0002, 0x319b: 0x0002, 0x319c: 0x0002, 0x319d: 0x0002, + 0x319e: 0x0002, 0x319f: 0x0002, 0x31a0: 0x0002, 0x31a1: 0x0002, 0x31a2: 0x0002, 0x31a3: 0x0002, + 0x31a4: 0x0002, 0x31a5: 0x0002, 0x31a6: 0x0002, 0x31a7: 0x0002, 0x31a8: 0x0002, 0x31a9: 0x0002, + 0x31aa: 0x0002, 0x31ab: 0x0002, 0x31ac: 0x0002, 0x31ad: 0x0002, 0x31ae: 0x0002, 0x31af: 0x0002, + 0x31b0: 0x0002, 0x31b1: 0x0002, 0x31b2: 0x0002, 0x31b3: 0x0002, 0x31b4: 0x0002, 0x31b5: 0x0002, + 0x31b6: 0x0002, 0x31b7: 0x0002, 0x31b8: 0x0002, 0x31b9: 0x0002, 0x31ba: 0x0002, 0x31bb: 0x0002, + 0x31bc: 0x0002, 0x31bd: 0x0002, 0x31be: 0x0002, 0x31bf: 0x0002, + // Block 0xc7, offset 0x31c0 + 0x31c0: 0x000c, 0x31c1: 0x000c, 0x31c2: 0x000c, 0x31c3: 0x000c, 0x31c4: 0x000c, 0x31c5: 0x000c, + 0x31c6: 0x000c, 0x31c7: 0x000c, 0x31c8: 0x000c, 0x31c9: 0x000c, 0x31ca: 0x000c, 0x31cb: 0x000c, + 0x31cc: 0x000c, 0x31cd: 0x000c, 0x31ce: 0x000c, 0x31cf: 0x000c, 0x31d0: 0x000c, 0x31d1: 0x000c, + 0x31d2: 0x000c, 0x31d3: 0x000c, 0x31d4: 0x000c, 0x31d5: 0x000c, 0x31d6: 0x000c, 0x31d7: 0x000c, + 0x31d8: 0x000c, 0x31d9: 0x000c, 0x31da: 0x000c, 0x31db: 0x000c, 0x31dc: 0x000c, 0x31dd: 0x000c, + 0x31de: 0x000c, 0x31df: 0x000c, 0x31e0: 0x000c, 0x31e1: 0x000c, 0x31e2: 0x000c, 0x31e3: 0x000c, + 0x31e4: 0x000c, 0x31e5: 0x000c, 0x31e6: 0x000c, 0x31e7: 0x000c, 0x31e8: 0x000c, 0x31e9: 0x000c, + 0x31ea: 0x000c, 0x31eb: 0x000c, 0x31ec: 0x000c, 0x31ed: 0x000c, 0x31ee: 0x000c, 0x31ef: 0x000c, + 0x31f0: 0x000c, 0x31f1: 0x000c, 0x31f2: 0x000c, 0x31f3: 0x000c, 0x31f4: 0x000c, 0x31f5: 0x000c, + 0x31f6: 0x000c, 0x31fb: 0x000c, + 0x31fc: 0x000c, 0x31fd: 0x000c, 0x31fe: 0x000c, 0x31ff: 0x000c, + // Block 0xc8, offset 0x3200 + 0x3200: 0x000c, 0x3201: 0x000c, 0x3202: 0x000c, 0x3203: 0x000c, 0x3204: 0x000c, 0x3205: 0x000c, + 0x3206: 0x000c, 0x3207: 0x000c, 0x3208: 0x000c, 0x3209: 0x000c, 0x320a: 0x000c, 0x320b: 0x000c, + 0x320c: 0x000c, 0x320d: 0x000c, 0x320e: 0x000c, 0x320f: 0x000c, 0x3210: 0x000c, 0x3211: 0x000c, + 0x3212: 0x000c, 0x3213: 0x000c, 0x3214: 0x000c, 0x3215: 0x000c, 0x3216: 0x000c, 0x3217: 0x000c, + 0x3218: 0x000c, 0x3219: 0x000c, 0x321a: 0x000c, 0x321b: 0x000c, 0x321c: 0x000c, 0x321d: 0x000c, + 0x321e: 0x000c, 0x321f: 0x000c, 0x3220: 0x000c, 0x3221: 0x000c, 0x3222: 0x000c, 0x3223: 0x000c, + 0x3224: 0x000c, 0x3225: 0x000c, 0x3226: 0x000c, 0x3227: 0x000c, 0x3228: 0x000c, 0x3229: 0x000c, + 0x322a: 0x000c, 0x322b: 0x000c, 0x322c: 0x000c, + 0x3235: 0x000c, + // Block 0xc9, offset 0x3240 + 0x3244: 0x000c, + 0x325b: 0x000c, 0x325c: 0x000c, 0x325d: 0x000c, + 0x325e: 0x000c, 0x325f: 0x000c, 0x3261: 0x000c, 0x3262: 0x000c, 0x3263: 0x000c, + 0x3264: 0x000c, 0x3265: 0x000c, 0x3266: 0x000c, 0x3267: 0x000c, 0x3268: 0x000c, 0x3269: 0x000c, + 0x326a: 0x000c, 0x326b: 0x000c, 0x326c: 0x000c, 0x326d: 0x000c, 0x326e: 0x000c, 0x326f: 0x000c, + // Block 0xca, offset 0x3280 + 0x3280: 0x000c, 0x3281: 0x000c, 0x3282: 0x000c, 0x3283: 0x000c, 0x3284: 0x000c, 0x3285: 0x000c, + 0x3286: 0x000c, 0x3288: 0x000c, 0x3289: 0x000c, 0x328a: 0x000c, 0x328b: 0x000c, + 0x328c: 0x000c, 0x328d: 0x000c, 0x328e: 0x000c, 0x328f: 0x000c, 0x3290: 0x000c, 0x3291: 0x000c, + 0x3292: 0x000c, 0x3293: 0x000c, 0x3294: 0x000c, 0x3295: 0x000c, 0x3296: 0x000c, 0x3297: 0x000c, + 0x3298: 0x000c, 0x329b: 0x000c, 0x329c: 0x000c, 0x329d: 0x000c, + 0x329e: 0x000c, 0x329f: 0x000c, 0x32a0: 0x000c, 0x32a1: 0x000c, 0x32a3: 0x000c, + 0x32a4: 0x000c, 0x32a6: 0x000c, 0x32a7: 0x000c, 0x32a8: 0x000c, 0x32a9: 0x000c, + 0x32aa: 0x000c, + // Block 0xcb, offset 0x32c0 + 0x32c0: 0x0001, 0x32c1: 0x0001, 0x32c2: 0x0001, 0x32c3: 0x0001, 0x32c4: 0x0001, 0x32c5: 0x0001, + 0x32c6: 0x0001, 0x32c7: 0x0001, 0x32c8: 0x0001, 0x32c9: 0x0001, 0x32ca: 0x0001, 0x32cb: 0x0001, + 0x32cc: 0x0001, 0x32cd: 0x0001, 0x32ce: 0x0001, 0x32cf: 0x0001, 0x32d0: 0x000c, 0x32d1: 0x000c, + 0x32d2: 0x000c, 0x32d3: 0x000c, 0x32d4: 0x000c, 0x32d5: 0x000c, 0x32d6: 0x000c, 0x32d7: 0x0001, + 0x32d8: 0x0001, 0x32d9: 0x0001, 0x32da: 0x0001, 0x32db: 0x0001, 0x32dc: 0x0001, 0x32dd: 0x0001, + 0x32de: 0x0001, 0x32df: 0x0001, 0x32e0: 0x0001, 0x32e1: 0x0001, 0x32e2: 0x0001, 0x32e3: 0x0001, + 0x32e4: 0x0001, 0x32e5: 0x0001, 0x32e6: 0x0001, 0x32e7: 0x0001, 0x32e8: 0x0001, 0x32e9: 0x0001, + 0x32ea: 0x0001, 0x32eb: 0x0001, 0x32ec: 0x0001, 0x32ed: 0x0001, 0x32ee: 0x0001, 0x32ef: 0x0001, + 0x32f0: 0x0001, 0x32f1: 0x0001, 0x32f2: 0x0001, 0x32f3: 0x0001, 0x32f4: 0x0001, 0x32f5: 0x0001, + 0x32f6: 0x0001, 0x32f7: 0x0001, 0x32f8: 0x0001, 0x32f9: 0x0001, 0x32fa: 0x0001, 0x32fb: 0x0001, + 0x32fc: 0x0001, 0x32fd: 0x0001, 0x32fe: 0x0001, 0x32ff: 0x0001, + // Block 0xcc, offset 0x3300 + 0x3300: 0x0001, 0x3301: 0x0001, 0x3302: 0x0001, 0x3303: 0x0001, 0x3304: 0x000c, 0x3305: 0x000c, + 0x3306: 0x000c, 0x3307: 0x000c, 0x3308: 0x000c, 0x3309: 0x000c, 0x330a: 0x000c, 0x330b: 0x0001, + 0x330c: 0x0001, 0x330d: 0x0001, 0x330e: 0x0001, 0x330f: 0x0001, 0x3310: 0x0001, 0x3311: 0x0001, + 0x3312: 0x0001, 0x3313: 0x0001, 0x3314: 0x0001, 0x3315: 0x0001, 0x3316: 0x0001, 0x3317: 0x0001, + 0x3318: 0x0001, 0x3319: 0x0001, 0x331a: 0x0001, 0x331b: 0x0001, 0x331c: 0x0001, 0x331d: 0x0001, + 0x331e: 0x0001, 0x331f: 0x0001, 0x3320: 0x0001, 0x3321: 0x0001, 0x3322: 0x0001, 0x3323: 0x0001, + 0x3324: 0x0001, 0x3325: 0x0001, 0x3326: 0x0001, 0x3327: 0x0001, 0x3328: 0x0001, 0x3329: 0x0001, + 0x332a: 0x0001, 0x332b: 0x0001, 0x332c: 0x0001, 0x332d: 0x0001, 0x332e: 0x0001, 0x332f: 0x0001, + 0x3330: 0x0001, 0x3331: 0x0001, 0x3332: 0x0001, 0x3333: 0x0001, 0x3334: 0x0001, 0x3335: 0x0001, + 0x3336: 0x0001, 0x3337: 0x0001, 0x3338: 0x0001, 0x3339: 0x0001, 0x333a: 0x0001, 0x333b: 0x0001, + 0x333c: 0x0001, 0x333d: 0x0001, 0x333e: 0x0001, 0x333f: 0x0001, + // Block 0xcd, offset 0x3340 + 0x3340: 0x000d, 0x3341: 0x000d, 0x3342: 0x000d, 0x3343: 0x000d, 0x3344: 0x000d, 0x3345: 0x000d, + 0x3346: 0x000d, 0x3347: 0x000d, 0x3348: 0x000d, 0x3349: 0x000d, 0x334a: 0x000d, 0x334b: 0x000d, + 0x334c: 0x000d, 0x334d: 0x000d, 0x334e: 0x000d, 0x334f: 0x000d, 0x3350: 0x000d, 0x3351: 0x000d, + 0x3352: 0x000d, 0x3353: 0x000d, 0x3354: 0x000d, 0x3355: 0x000d, 0x3356: 0x000d, 0x3357: 0x000d, + 0x3358: 0x000d, 0x3359: 0x000d, 0x335a: 0x000d, 0x335b: 0x000d, 0x335c: 0x000d, 0x335d: 0x000d, + 0x335e: 0x000d, 0x335f: 0x000d, 0x3360: 0x000d, 0x3361: 0x000d, 0x3362: 0x000d, 0x3363: 0x000d, + 0x3364: 0x000d, 0x3365: 0x000d, 0x3366: 0x000d, 0x3367: 0x000d, 0x3368: 0x000d, 0x3369: 0x000d, + 0x336a: 0x000d, 0x336b: 0x000d, 0x336c: 0x000d, 0x336d: 0x000d, 0x336e: 0x000d, 0x336f: 0x000d, + 0x3370: 0x000a, 0x3371: 0x000a, 0x3372: 0x000d, 0x3373: 0x000d, 0x3374: 0x000d, 0x3375: 0x000d, + 0x3376: 0x000d, 0x3377: 0x000d, 0x3378: 0x000d, 0x3379: 0x000d, 0x337a: 0x000d, 0x337b: 0x000d, + 0x337c: 0x000d, 0x337d: 0x000d, 0x337e: 0x000d, 0x337f: 0x000d, + // Block 0xce, offset 0x3380 + 0x3380: 0x000a, 0x3381: 0x000a, 0x3382: 0x000a, 0x3383: 0x000a, 0x3384: 0x000a, 0x3385: 0x000a, + 0x3386: 0x000a, 0x3387: 0x000a, 0x3388: 0x000a, 0x3389: 0x000a, 0x338a: 0x000a, 0x338b: 0x000a, + 0x338c: 0x000a, 0x338d: 0x000a, 0x338e: 0x000a, 0x338f: 0x000a, 0x3390: 0x000a, 0x3391: 0x000a, + 0x3392: 0x000a, 0x3393: 0x000a, 0x3394: 0x000a, 0x3395: 0x000a, 0x3396: 0x000a, 0x3397: 0x000a, + 0x3398: 0x000a, 0x3399: 0x000a, 0x339a: 0x000a, 0x339b: 0x000a, 0x339c: 0x000a, 0x339d: 0x000a, + 0x339e: 0x000a, 0x339f: 0x000a, 0x33a0: 0x000a, 0x33a1: 0x000a, 0x33a2: 0x000a, 0x33a3: 0x000a, + 0x33a4: 0x000a, 0x33a5: 0x000a, 0x33a6: 0x000a, 0x33a7: 0x000a, 0x33a8: 0x000a, 0x33a9: 0x000a, + 0x33aa: 0x000a, 0x33ab: 0x000a, + 0x33b0: 0x000a, 0x33b1: 0x000a, 0x33b2: 0x000a, 0x33b3: 0x000a, 0x33b4: 0x000a, 0x33b5: 0x000a, + 0x33b6: 0x000a, 0x33b7: 0x000a, 0x33b8: 0x000a, 0x33b9: 0x000a, 0x33ba: 0x000a, 0x33bb: 0x000a, + 0x33bc: 0x000a, 0x33bd: 0x000a, 0x33be: 0x000a, 0x33bf: 0x000a, + // Block 0xcf, offset 0x33c0 + 0x33c0: 0x000a, 0x33c1: 0x000a, 0x33c2: 0x000a, 0x33c3: 0x000a, 0x33c4: 0x000a, 0x33c5: 0x000a, + 0x33c6: 0x000a, 0x33c7: 0x000a, 0x33c8: 0x000a, 0x33c9: 0x000a, 0x33ca: 0x000a, 0x33cb: 0x000a, + 0x33cc: 0x000a, 0x33cd: 0x000a, 0x33ce: 0x000a, 0x33cf: 0x000a, 0x33d0: 0x000a, 0x33d1: 0x000a, + 0x33d2: 0x000a, 0x33d3: 0x000a, + 0x33e0: 0x000a, 0x33e1: 0x000a, 0x33e2: 0x000a, 0x33e3: 0x000a, + 0x33e4: 0x000a, 0x33e5: 0x000a, 0x33e6: 0x000a, 0x33e7: 0x000a, 0x33e8: 0x000a, 0x33e9: 0x000a, + 0x33ea: 0x000a, 0x33eb: 0x000a, 0x33ec: 0x000a, 0x33ed: 0x000a, 0x33ee: 0x000a, + 0x33f1: 0x000a, 0x33f2: 0x000a, 0x33f3: 0x000a, 0x33f4: 0x000a, 0x33f5: 0x000a, + 0x33f6: 0x000a, 0x33f7: 0x000a, 0x33f8: 0x000a, 0x33f9: 0x000a, 0x33fa: 0x000a, 0x33fb: 0x000a, + 0x33fc: 0x000a, 0x33fd: 0x000a, 0x33fe: 0x000a, 0x33ff: 0x000a, + // Block 0xd0, offset 0x3400 + 0x3401: 0x000a, 0x3402: 0x000a, 0x3403: 0x000a, 0x3404: 0x000a, 0x3405: 0x000a, + 0x3406: 0x000a, 0x3407: 0x000a, 0x3408: 0x000a, 0x3409: 0x000a, 0x340a: 0x000a, 0x340b: 0x000a, + 0x340c: 0x000a, 0x340d: 0x000a, 0x340e: 0x000a, 0x340f: 0x000a, 0x3411: 0x000a, + 0x3412: 0x000a, 0x3413: 0x000a, 0x3414: 0x000a, 0x3415: 0x000a, 0x3416: 0x000a, 0x3417: 0x000a, + 0x3418: 0x000a, 0x3419: 0x000a, 0x341a: 0x000a, 0x341b: 0x000a, 0x341c: 0x000a, 0x341d: 0x000a, + 0x341e: 0x000a, 0x341f: 0x000a, 0x3420: 0x000a, 0x3421: 0x000a, 0x3422: 0x000a, 0x3423: 0x000a, + 0x3424: 0x000a, 0x3425: 0x000a, 0x3426: 0x000a, 0x3427: 0x000a, 0x3428: 0x000a, 0x3429: 0x000a, + 0x342a: 0x000a, 0x342b: 0x000a, 0x342c: 0x000a, 0x342d: 0x000a, 0x342e: 0x000a, 0x342f: 0x000a, + 0x3430: 0x000a, 0x3431: 0x000a, 0x3432: 0x000a, 0x3433: 0x000a, 0x3434: 0x000a, 0x3435: 0x000a, + // Block 0xd1, offset 0x3440 + 0x3440: 0x0002, 0x3441: 0x0002, 0x3442: 0x0002, 0x3443: 0x0002, 0x3444: 0x0002, 0x3445: 0x0002, + 0x3446: 0x0002, 0x3447: 0x0002, 0x3448: 0x0002, 0x3449: 0x0002, 0x344a: 0x0002, 0x344b: 0x000a, + 0x344c: 0x000a, + // Block 0xd2, offset 0x3480 + 0x34aa: 0x000a, 0x34ab: 0x000a, + // Block 0xd3, offset 0x34c0 + 0x34c0: 0x000a, 0x34c1: 0x000a, 0x34c2: 0x000a, 0x34c3: 0x000a, 0x34c4: 0x000a, 0x34c5: 0x000a, + 0x34c6: 0x000a, 0x34c7: 0x000a, 0x34c8: 0x000a, 0x34c9: 0x000a, 0x34ca: 0x000a, 0x34cb: 0x000a, + 0x34cc: 0x000a, 0x34cd: 0x000a, 0x34ce: 0x000a, 0x34cf: 0x000a, 0x34d0: 0x000a, 0x34d1: 0x000a, + 0x34d2: 0x000a, + 0x34e0: 0x000a, 0x34e1: 0x000a, 0x34e2: 0x000a, 0x34e3: 0x000a, + 0x34e4: 0x000a, 0x34e5: 0x000a, 0x34e6: 0x000a, 0x34e7: 0x000a, 0x34e8: 0x000a, 0x34e9: 0x000a, + 0x34ea: 0x000a, 0x34eb: 0x000a, 0x34ec: 0x000a, + 0x34f0: 0x000a, 0x34f1: 0x000a, 0x34f2: 0x000a, 0x34f3: 0x000a, 0x34f4: 0x000a, 0x34f5: 0x000a, + 0x34f6: 0x000a, + // Block 0xd4, offset 0x3500 + 0x3500: 0x000a, 0x3501: 0x000a, 0x3502: 0x000a, 0x3503: 0x000a, 0x3504: 0x000a, 0x3505: 0x000a, + 0x3506: 0x000a, 0x3507: 0x000a, 0x3508: 0x000a, 0x3509: 0x000a, 0x350a: 0x000a, 0x350b: 0x000a, + 0x350c: 0x000a, 0x350d: 0x000a, 0x350e: 0x000a, 0x350f: 0x000a, 0x3510: 0x000a, 0x3511: 0x000a, + 0x3512: 0x000a, 0x3513: 0x000a, 0x3514: 0x000a, + // Block 0xd5, offset 0x3540 + 0x3540: 0x000a, 0x3541: 0x000a, 0x3542: 0x000a, 0x3543: 0x000a, 0x3544: 0x000a, 0x3545: 0x000a, + 0x3546: 0x000a, 0x3547: 0x000a, 0x3548: 0x000a, 0x3549: 0x000a, 0x354a: 0x000a, 0x354b: 0x000a, + 0x3550: 0x000a, 0x3551: 0x000a, + 0x3552: 0x000a, 0x3553: 0x000a, 0x3554: 0x000a, 0x3555: 0x000a, 0x3556: 0x000a, 0x3557: 0x000a, + 0x3558: 0x000a, 0x3559: 0x000a, 0x355a: 0x000a, 0x355b: 0x000a, 0x355c: 0x000a, 0x355d: 0x000a, + 0x355e: 0x000a, 0x355f: 0x000a, 0x3560: 0x000a, 0x3561: 0x000a, 0x3562: 0x000a, 0x3563: 0x000a, + 0x3564: 0x000a, 0x3565: 0x000a, 0x3566: 0x000a, 0x3567: 0x000a, 0x3568: 0x000a, 0x3569: 0x000a, + 0x356a: 0x000a, 0x356b: 0x000a, 0x356c: 0x000a, 0x356d: 0x000a, 0x356e: 0x000a, 0x356f: 0x000a, + 0x3570: 0x000a, 0x3571: 0x000a, 0x3572: 0x000a, 0x3573: 0x000a, 0x3574: 0x000a, 0x3575: 0x000a, + 0x3576: 0x000a, 0x3577: 0x000a, 0x3578: 0x000a, 0x3579: 0x000a, 0x357a: 0x000a, 0x357b: 0x000a, + 0x357c: 0x000a, 0x357d: 0x000a, 0x357e: 0x000a, 0x357f: 0x000a, + // Block 0xd6, offset 0x3580 + 0x3580: 0x000a, 0x3581: 0x000a, 0x3582: 0x000a, 0x3583: 0x000a, 0x3584: 0x000a, 0x3585: 0x000a, + 0x3586: 0x000a, 0x3587: 0x000a, + 0x3590: 0x000a, 0x3591: 0x000a, + 0x3592: 0x000a, 0x3593: 0x000a, 0x3594: 0x000a, 0x3595: 0x000a, 0x3596: 0x000a, 0x3597: 0x000a, + 0x3598: 0x000a, 0x3599: 0x000a, + 0x35a0: 0x000a, 0x35a1: 0x000a, 0x35a2: 0x000a, 0x35a3: 0x000a, + 0x35a4: 0x000a, 0x35a5: 0x000a, 0x35a6: 0x000a, 0x35a7: 0x000a, 0x35a8: 0x000a, 0x35a9: 0x000a, + 0x35aa: 0x000a, 0x35ab: 0x000a, 0x35ac: 0x000a, 0x35ad: 0x000a, 0x35ae: 0x000a, 0x35af: 0x000a, + 0x35b0: 0x000a, 0x35b1: 0x000a, 0x35b2: 0x000a, 0x35b3: 0x000a, 0x35b4: 0x000a, 0x35b5: 0x000a, + 0x35b6: 0x000a, 0x35b7: 0x000a, 0x35b8: 0x000a, 0x35b9: 0x000a, 0x35ba: 0x000a, 0x35bb: 0x000a, + 0x35bc: 0x000a, 0x35bd: 0x000a, 0x35be: 0x000a, 0x35bf: 0x000a, + // Block 0xd7, offset 0x35c0 + 0x35c0: 0x000a, 0x35c1: 0x000a, 0x35c2: 0x000a, 0x35c3: 0x000a, 0x35c4: 0x000a, 0x35c5: 0x000a, + 0x35c6: 0x000a, 0x35c7: 0x000a, + 0x35d0: 0x000a, 0x35d1: 0x000a, + 0x35d2: 0x000a, 0x35d3: 0x000a, 0x35d4: 0x000a, 0x35d5: 0x000a, 0x35d6: 0x000a, 0x35d7: 0x000a, + 0x35d8: 0x000a, 0x35d9: 0x000a, 0x35da: 0x000a, 0x35db: 0x000a, 0x35dc: 0x000a, 0x35dd: 0x000a, + 0x35de: 0x000a, 0x35df: 0x000a, 0x35e0: 0x000a, 0x35e1: 0x000a, 0x35e2: 0x000a, 0x35e3: 0x000a, + 0x35e4: 0x000a, 0x35e5: 0x000a, 0x35e6: 0x000a, 0x35e7: 0x000a, 0x35e8: 0x000a, 0x35e9: 0x000a, + 0x35ea: 0x000a, 0x35eb: 0x000a, 0x35ec: 0x000a, 0x35ed: 0x000a, + // Block 0xd8, offset 0x3600 + 0x3610: 0x000a, 0x3611: 0x000a, + 0x3612: 0x000a, 0x3613: 0x000a, 0x3614: 0x000a, 0x3615: 0x000a, 0x3616: 0x000a, 0x3617: 0x000a, + 0x3618: 0x000a, 0x3619: 0x000a, 0x361a: 0x000a, 0x361b: 0x000a, 0x361c: 0x000a, 0x361d: 0x000a, + 0x361e: 0x000a, 0x3620: 0x000a, 0x3621: 0x000a, 0x3622: 0x000a, 0x3623: 0x000a, + 0x3624: 0x000a, 0x3625: 0x000a, 0x3626: 0x000a, 0x3627: 0x000a, + 0x3630: 0x000a, 0x3633: 0x000a, 0x3634: 0x000a, 0x3635: 0x000a, + 0x3636: 0x000a, 0x3637: 0x000a, 0x3638: 0x000a, 0x3639: 0x000a, 0x363a: 0x000a, 0x363b: 0x000a, + 0x363c: 0x000a, 0x363d: 0x000a, 0x363e: 0x000a, + // Block 0xd9, offset 0x3640 + 0x3640: 0x000a, 0x3641: 0x000a, 0x3642: 0x000a, 0x3643: 0x000a, 0x3644: 0x000a, 0x3645: 0x000a, + 0x3646: 0x000a, 0x3647: 0x000a, 0x3648: 0x000a, 0x3649: 0x000a, 0x364a: 0x000a, 0x364b: 0x000a, + 0x3650: 0x000a, 0x3651: 0x000a, + 0x3652: 0x000a, 0x3653: 0x000a, 0x3654: 0x000a, 0x3655: 0x000a, 0x3656: 0x000a, 0x3657: 0x000a, + 0x3658: 0x000a, 0x3659: 0x000a, 0x365a: 0x000a, 0x365b: 0x000a, 0x365c: 0x000a, 0x365d: 0x000a, + 0x365e: 0x000a, + // Block 0xda, offset 0x3680 + 0x3680: 0x000a, 0x3681: 0x000a, 0x3682: 0x000a, 0x3683: 0x000a, 0x3684: 0x000a, 0x3685: 0x000a, + 0x3686: 0x000a, 0x3687: 0x000a, 0x3688: 0x000a, 0x3689: 0x000a, 0x368a: 0x000a, 0x368b: 0x000a, + 0x368c: 0x000a, 0x368d: 0x000a, 0x368e: 0x000a, 0x368f: 0x000a, 0x3690: 0x000a, 0x3691: 0x000a, + // Block 0xdb, offset 0x36c0 + 0x36fe: 0x000b, 0x36ff: 0x000b, + // Block 0xdc, offset 0x3700 + 0x3700: 0x000b, 0x3701: 0x000b, 0x3702: 0x000b, 0x3703: 0x000b, 0x3704: 0x000b, 0x3705: 0x000b, + 0x3706: 0x000b, 0x3707: 0x000b, 0x3708: 0x000b, 0x3709: 0x000b, 0x370a: 0x000b, 0x370b: 0x000b, + 0x370c: 0x000b, 0x370d: 0x000b, 0x370e: 0x000b, 0x370f: 0x000b, 0x3710: 0x000b, 0x3711: 0x000b, + 0x3712: 0x000b, 0x3713: 0x000b, 0x3714: 0x000b, 0x3715: 0x000b, 0x3716: 0x000b, 0x3717: 0x000b, + 0x3718: 0x000b, 0x3719: 0x000b, 0x371a: 0x000b, 0x371b: 0x000b, 0x371c: 0x000b, 0x371d: 0x000b, + 0x371e: 0x000b, 0x371f: 0x000b, 0x3720: 0x000b, 0x3721: 0x000b, 0x3722: 0x000b, 0x3723: 0x000b, + 0x3724: 0x000b, 0x3725: 0x000b, 0x3726: 0x000b, 0x3727: 0x000b, 0x3728: 0x000b, 0x3729: 0x000b, + 0x372a: 0x000b, 0x372b: 0x000b, 0x372c: 0x000b, 0x372d: 0x000b, 0x372e: 0x000b, 0x372f: 0x000b, + 0x3730: 0x000b, 0x3731: 0x000b, 0x3732: 0x000b, 0x3733: 0x000b, 0x3734: 0x000b, 0x3735: 0x000b, + 0x3736: 0x000b, 0x3737: 0x000b, 0x3738: 0x000b, 0x3739: 0x000b, 0x373a: 0x000b, 0x373b: 0x000b, + 0x373c: 0x000b, 0x373d: 0x000b, 0x373e: 0x000b, 0x373f: 0x000b, + // Block 0xdd, offset 0x3740 + 0x3740: 0x000c, 0x3741: 0x000c, 0x3742: 0x000c, 0x3743: 0x000c, 0x3744: 0x000c, 0x3745: 0x000c, + 0x3746: 0x000c, 0x3747: 0x000c, 0x3748: 0x000c, 0x3749: 0x000c, 0x374a: 0x000c, 0x374b: 0x000c, + 0x374c: 0x000c, 0x374d: 0x000c, 0x374e: 0x000c, 0x374f: 0x000c, 0x3750: 0x000c, 0x3751: 0x000c, + 0x3752: 0x000c, 0x3753: 0x000c, 0x3754: 0x000c, 0x3755: 0x000c, 0x3756: 0x000c, 0x3757: 0x000c, + 0x3758: 0x000c, 0x3759: 0x000c, 0x375a: 0x000c, 0x375b: 0x000c, 0x375c: 0x000c, 0x375d: 0x000c, + 0x375e: 0x000c, 0x375f: 0x000c, 0x3760: 0x000c, 0x3761: 0x000c, 0x3762: 0x000c, 0x3763: 0x000c, + 0x3764: 0x000c, 0x3765: 0x000c, 0x3766: 0x000c, 0x3767: 0x000c, 0x3768: 0x000c, 0x3769: 0x000c, + 0x376a: 0x000c, 0x376b: 0x000c, 0x376c: 0x000c, 0x376d: 0x000c, 0x376e: 0x000c, 0x376f: 0x000c, + 0x3770: 0x000b, 0x3771: 0x000b, 0x3772: 0x000b, 0x3773: 0x000b, 0x3774: 0x000b, 0x3775: 0x000b, + 0x3776: 0x000b, 0x3777: 0x000b, 0x3778: 0x000b, 0x3779: 0x000b, 0x377a: 0x000b, 0x377b: 0x000b, + 0x377c: 0x000b, 0x377d: 0x000b, 0x377e: 0x000b, 0x377f: 0x000b, +} + +// bidiIndex: 24 blocks, 1536 entries, 1536 bytes +// Block 0 is the zero block. +var bidiIndex = [1536]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x02, + 0xca: 0x03, 0xcb: 0x04, 0xcc: 0x05, 0xcd: 0x06, 0xce: 0x07, 0xcf: 0x08, + 0xd2: 0x09, 0xd6: 0x0a, 0xd7: 0x0b, + 0xd8: 0x0c, 0xd9: 0x0d, 0xda: 0x0e, 0xdb: 0x0f, 0xdc: 0x10, 0xdd: 0x11, 0xde: 0x12, 0xdf: 0x13, + 0xe0: 0x02, 0xe1: 0x03, 0xe2: 0x04, 0xe3: 0x05, 0xe4: 0x06, + 0xea: 0x07, 0xef: 0x08, + 0xf0: 0x11, 0xf1: 0x12, 0xf2: 0x12, 0xf3: 0x14, 0xf4: 0x15, + // Block 0x4, offset 0x100 + 0x120: 0x14, 0x121: 0x15, 0x122: 0x16, 0x123: 0x17, 0x124: 0x18, 0x125: 0x19, 0x126: 0x1a, 0x127: 0x1b, + 0x128: 0x1c, 0x129: 0x1d, 0x12a: 0x1c, 0x12b: 0x1e, 0x12c: 0x1f, 0x12d: 0x20, 0x12e: 0x21, 0x12f: 0x22, + 0x130: 0x23, 0x131: 0x24, 0x132: 0x1a, 0x133: 0x25, 0x134: 0x26, 0x135: 0x27, 0x137: 0x28, + 0x138: 0x29, 0x139: 0x2a, 0x13a: 0x2b, 0x13b: 0x2c, 0x13c: 0x2d, 0x13d: 0x2e, 0x13e: 0x2f, 0x13f: 0x30, + // Block 0x5, offset 0x140 + 0x140: 0x31, 0x141: 0x32, 0x142: 0x33, + 0x14d: 0x34, 0x14e: 0x35, + 0x150: 0x36, + 0x15a: 0x37, 0x15c: 0x38, 0x15d: 0x39, 0x15e: 0x3a, 0x15f: 0x3b, + 0x160: 0x3c, 0x162: 0x3d, 0x164: 0x3e, 0x165: 0x3f, 0x167: 0x40, + 0x168: 0x41, 0x169: 0x42, 0x16a: 0x43, 0x16c: 0x44, 0x16d: 0x45, 0x16e: 0x46, 0x16f: 0x47, + 0x170: 0x48, 0x173: 0x49, 0x177: 0x4a, + 0x17e: 0x4b, 0x17f: 0x4c, + // Block 0x6, offset 0x180 + 0x180: 0x4d, 0x181: 0x4e, 0x182: 0x4f, 0x183: 0x50, 0x184: 0x51, 0x185: 0x52, 0x186: 0x53, 0x187: 0x54, + 0x188: 0x55, 0x189: 0x54, 0x18a: 0x54, 0x18b: 0x54, 0x18c: 0x56, 0x18d: 0x57, 0x18e: 0x58, 0x18f: 0x59, + 0x190: 0x5a, 0x191: 0x5b, 0x192: 0x5c, 0x193: 0x5d, 0x194: 0x54, 0x195: 0x54, 0x196: 0x54, 0x197: 0x54, + 0x198: 0x54, 0x199: 0x54, 0x19a: 0x5e, 0x19b: 0x54, 0x19c: 0x54, 0x19d: 0x5f, 0x19e: 0x54, 0x19f: 0x60, + 0x1a4: 0x54, 0x1a5: 0x54, 0x1a6: 0x61, 0x1a7: 0x62, + 0x1a8: 0x54, 0x1a9: 0x54, 0x1aa: 0x54, 0x1ab: 0x54, 0x1ac: 0x54, 0x1ad: 0x63, 0x1ae: 0x64, 0x1af: 0x65, + 0x1b3: 0x66, 0x1b5: 0x67, 0x1b7: 0x68, + 0x1b8: 0x69, 0x1b9: 0x6a, 0x1ba: 0x6b, 0x1bb: 0x6c, 0x1bc: 0x54, 0x1bd: 0x54, 0x1be: 0x54, 0x1bf: 0x6d, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x6e, 0x1c2: 0x6f, 0x1c3: 0x70, 0x1c7: 0x71, + 0x1c8: 0x72, 0x1c9: 0x73, 0x1ca: 0x74, 0x1cb: 0x75, 0x1cd: 0x76, 0x1cf: 0x77, + // Block 0x8, offset 0x200 + 0x237: 0x54, + // Block 0x9, offset 0x240 + 0x252: 0x78, 0x253: 0x79, + 0x258: 0x7a, 0x259: 0x7b, 0x25a: 0x7c, 0x25b: 0x7d, 0x25c: 0x7e, 0x25e: 0x7f, + 0x260: 0x80, 0x261: 0x81, 0x263: 0x82, 0x264: 0x83, 0x265: 0x84, 0x266: 0x85, 0x267: 0x86, + 0x268: 0x87, 0x269: 0x88, 0x26a: 0x89, 0x26b: 0x8a, 0x26f: 0x8b, + // Block 0xa, offset 0x280 + 0x2ac: 0x8c, 0x2ad: 0x8d, 0x2ae: 0x0e, 0x2af: 0x0e, + 0x2b0: 0x0e, 0x2b1: 0x0e, 0x2b2: 0x0e, 0x2b3: 0x0e, 0x2b4: 0x8e, 0x2b5: 0x0e, 0x2b6: 0x0e, 0x2b7: 0x8f, + 0x2b8: 0x90, 0x2b9: 0x91, 0x2ba: 0x0e, 0x2bb: 0x92, 0x2bc: 0x93, 0x2bd: 0x94, 0x2bf: 0x95, + // Block 0xb, offset 0x2c0 + 0x2c4: 0x96, 0x2c5: 0x54, 0x2c6: 0x97, 0x2c7: 0x98, + 0x2cb: 0x99, 0x2cd: 0x9a, + 0x2e0: 0x9b, 0x2e1: 0x9b, 0x2e2: 0x9b, 0x2e3: 0x9b, 0x2e4: 0x9c, 0x2e5: 0x9b, 0x2e6: 0x9b, 0x2e7: 0x9b, + 0x2e8: 0x9d, 0x2e9: 0x9b, 0x2ea: 0x9b, 0x2eb: 0x9e, 0x2ec: 0x9f, 0x2ed: 0x9b, 0x2ee: 0x9b, 0x2ef: 0x9b, + 0x2f0: 0x9b, 0x2f1: 0x9b, 0x2f2: 0x9b, 0x2f3: 0x9b, 0x2f4: 0x9b, 0x2f5: 0x9b, 0x2f6: 0x9b, 0x2f7: 0x9b, + 0x2f8: 0x9b, 0x2f9: 0xa0, 0x2fa: 0x9b, 0x2fb: 0x9b, 0x2fc: 0x9b, 0x2fd: 0x9b, 0x2fe: 0x9b, 0x2ff: 0x9b, + // Block 0xc, offset 0x300 + 0x300: 0xa1, 0x301: 0xa2, 0x302: 0xa3, 0x304: 0xa4, 0x305: 0xa5, 0x306: 0xa6, 0x307: 0xa7, + 0x308: 0xa8, 0x30b: 0xa9, 0x30c: 0xaa, 0x30d: 0xab, + 0x310: 0xac, 0x311: 0xad, 0x312: 0xae, 0x313: 0xaf, 0x316: 0xb0, 0x317: 0xb1, + 0x318: 0xb2, 0x319: 0xb3, 0x31a: 0xb4, 0x31c: 0xb5, + 0x330: 0xb6, 0x332: 0xb7, + // Block 0xd, offset 0x340 + 0x36b: 0xb8, 0x36c: 0xb9, + 0x37e: 0xba, + // Block 0xe, offset 0x380 + 0x3b2: 0xbb, + // Block 0xf, offset 0x3c0 + 0x3c5: 0xbc, 0x3c6: 0xbd, + 0x3c8: 0x54, 0x3c9: 0xbe, 0x3cc: 0x54, 0x3cd: 0xbf, + 0x3db: 0xc0, 0x3dc: 0xc1, 0x3dd: 0xc2, 0x3de: 0xc3, 0x3df: 0xc4, + 0x3e8: 0xc5, 0x3e9: 0xc6, 0x3ea: 0xc7, + // Block 0x10, offset 0x400 + 0x400: 0xc8, + 0x420: 0x9b, 0x421: 0x9b, 0x422: 0x9b, 0x423: 0xc9, 0x424: 0x9b, 0x425: 0xca, 0x426: 0x9b, 0x427: 0x9b, + 0x428: 0x9b, 0x429: 0x9b, 0x42a: 0x9b, 0x42b: 0x9b, 0x42c: 0x9b, 0x42d: 0x9b, 0x42e: 0x9b, 0x42f: 0x9b, + 0x430: 0x9b, 0x431: 0x9b, 0x432: 0x9b, 0x433: 0x9b, 0x434: 0x9b, 0x435: 0x9b, 0x436: 0x9b, 0x437: 0x9b, + 0x438: 0x0e, 0x439: 0x0e, 0x43a: 0x0e, 0x43b: 0xcb, 0x43c: 0x9b, 0x43d: 0x9b, 0x43e: 0x9b, 0x43f: 0x9b, + // Block 0x11, offset 0x440 + 0x440: 0xcc, 0x441: 0x54, 0x442: 0xcd, 0x443: 0xce, 0x444: 0xcf, 0x445: 0xd0, + 0x44c: 0x54, 0x44d: 0x54, 0x44e: 0x54, 0x44f: 0x54, + 0x450: 0x54, 0x451: 0x54, 0x452: 0x54, 0x453: 0x54, 0x454: 0x54, 0x455: 0x54, 0x456: 0x54, 0x457: 0x54, + 0x458: 0x54, 0x459: 0x54, 0x45a: 0x54, 0x45b: 0xd1, 0x45c: 0x54, 0x45d: 0x6c, 0x45e: 0x54, 0x45f: 0xd2, + 0x460: 0xd3, 0x461: 0xd4, 0x462: 0xd5, 0x464: 0xd6, 0x465: 0xd7, 0x466: 0xd8, 0x467: 0x36, + 0x47f: 0xd9, + // Block 0x12, offset 0x480 + 0x4bf: 0xd9, + // Block 0x13, offset 0x4c0 + 0x4d0: 0x09, 0x4d1: 0x0a, 0x4d6: 0x0b, + 0x4db: 0x0c, 0x4dd: 0x0d, 0x4de: 0x0e, 0x4df: 0x0f, + 0x4ef: 0x10, + 0x4ff: 0x10, + // Block 0x14, offset 0x500 + 0x50f: 0x10, + 0x51f: 0x10, + 0x52f: 0x10, + 0x53f: 0x10, + // Block 0x15, offset 0x540 + 0x540: 0xda, 0x541: 0xda, 0x542: 0xda, 0x543: 0xda, 0x544: 0x05, 0x545: 0x05, 0x546: 0x05, 0x547: 0xdb, + 0x548: 0xda, 0x549: 0xda, 0x54a: 0xda, 0x54b: 0xda, 0x54c: 0xda, 0x54d: 0xda, 0x54e: 0xda, 0x54f: 0xda, + 0x550: 0xda, 0x551: 0xda, 0x552: 0xda, 0x553: 0xda, 0x554: 0xda, 0x555: 0xda, 0x556: 0xda, 0x557: 0xda, + 0x558: 0xda, 0x559: 0xda, 0x55a: 0xda, 0x55b: 0xda, 0x55c: 0xda, 0x55d: 0xda, 0x55e: 0xda, 0x55f: 0xda, + 0x560: 0xda, 0x561: 0xda, 0x562: 0xda, 0x563: 0xda, 0x564: 0xda, 0x565: 0xda, 0x566: 0xda, 0x567: 0xda, + 0x568: 0xda, 0x569: 0xda, 0x56a: 0xda, 0x56b: 0xda, 0x56c: 0xda, 0x56d: 0xda, 0x56e: 0xda, 0x56f: 0xda, + 0x570: 0xda, 0x571: 0xda, 0x572: 0xda, 0x573: 0xda, 0x574: 0xda, 0x575: 0xda, 0x576: 0xda, 0x577: 0xda, + 0x578: 0xda, 0x579: 0xda, 0x57a: 0xda, 0x57b: 0xda, 0x57c: 0xda, 0x57d: 0xda, 0x57e: 0xda, 0x57f: 0xda, + // Block 0x16, offset 0x580 + 0x58f: 0x10, + 0x59f: 0x10, + 0x5a0: 0x13, + 0x5af: 0x10, + 0x5bf: 0x10, + // Block 0x17, offset 0x5c0 + 0x5cf: 0x10, +} + +// Total table size 15800 bytes (15KiB); checksum: F50EF68C diff --git a/vendor/golang.org/x/text/unicode/bidi/trieval.go b/vendor/golang.org/x/text/unicode/bidi/trieval.go new file mode 100644 index 00000000000..bebd855efde --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/trieval.go @@ -0,0 +1,60 @@ +// This file was generated by go generate; DO NOT EDIT + +package bidi + +// Class is the Unicode BiDi class. Each rune has a single class. +type Class uint + +const ( + L Class = iota // LeftToRight + R // RightToLeft + EN // EuropeanNumber + ES // EuropeanSeparator + ET // EuropeanTerminator + AN // ArabicNumber + CS // CommonSeparator + B // ParagraphSeparator + S // SegmentSeparator + WS // WhiteSpace + ON // OtherNeutral + BN // BoundaryNeutral + NSM // NonspacingMark + AL // ArabicLetter + Control // Control LRO - PDI + + numClass + + LRO // LeftToRightOverride + RLO // RightToLeftOverride + LRE // LeftToRightEmbedding + RLE // RightToLeftEmbedding + PDF // PopDirectionalFormat + LRI // LeftToRightIsolate + RLI // RightToLeftIsolate + FSI // FirstStrongIsolate + PDI // PopDirectionalIsolate + + unknownClass = ^Class(0) +) + +var controlToClass = map[rune]Class{ + 0x202D: LRO, // LeftToRightOverride, + 0x202E: RLO, // RightToLeftOverride, + 0x202A: LRE, // LeftToRightEmbedding, + 0x202B: RLE, // RightToLeftEmbedding, + 0x202C: PDF, // PopDirectionalFormat, + 0x2066: LRI, // LeftToRightIsolate, + 0x2067: RLI, // RightToLeftIsolate, + 0x2068: FSI, // FirstStrongIsolate, + 0x2069: PDI, // PopDirectionalIsolate, +} + +// A trie entry has the following bits: +// 7..5 XOR mask for brackets +// 4 1: Bracket open, 0: Bracket close +// 3..0 Class type + +const ( + openMask = 0x10 + xorMaskShift = 5 +) diff --git a/vendor/golang.org/x/text/unicode/norm/composition.go b/vendor/golang.org/x/text/unicode/norm/composition.go new file mode 100644 index 00000000000..d17b278adc3 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/composition.go @@ -0,0 +1,514 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import "unicode/utf8" + +const ( + maxNonStarters = 30 + // The maximum number of characters needed for a buffer is + // maxNonStarters + 1 for the starter + 1 for the GCJ + maxBufferSize = maxNonStarters + 2 + maxNFCExpansion = 3 // NFC(0x1D160) + maxNFKCExpansion = 18 // NFKC(0xFDFA) + + maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128 +) + +// ssState is used for reporting the segment state after inserting a rune. +// It is returned by streamSafe.next. +type ssState int + +const ( + // Indicates a rune was successfully added to the segment. + ssSuccess ssState = iota + // Indicates a rune starts a new segment and should not be added. + ssStarter + // Indicates a rune caused a segment overflow and a CGJ should be inserted. + ssOverflow +) + +// streamSafe implements the policy of when a CGJ should be inserted. +type streamSafe uint8 + +// mkStreamSafe is a shorthand for declaring a streamSafe var and calling +// first on it. +func mkStreamSafe(p Properties) streamSafe { + return streamSafe(p.nTrailingNonStarters()) +} + +// first inserts the first rune of a segment. +func (ss *streamSafe) first(p Properties) { + if *ss != 0 { + panic("!= 0") + } + *ss = streamSafe(p.nTrailingNonStarters()) +} + +// insert returns a ssState value to indicate whether a rune represented by p +// can be inserted. +func (ss *streamSafe) next(p Properties) ssState { + if *ss > maxNonStarters { + panic("streamSafe was not reset") + } + n := p.nLeadingNonStarters() + if *ss += streamSafe(n); *ss > maxNonStarters { + *ss = 0 + return ssOverflow + } + // The Stream-Safe Text Processing prescribes that the counting can stop + // as soon as a starter is encountered. However, there are some starters, + // like Jamo V and T, that can combine with other runes, leaving their + // successive non-starters appended to the previous, possibly causing an + // overflow. We will therefore consider any rune with a non-zero nLead to + // be a non-starter. Note that it always hold that if nLead > 0 then + // nLead == nTrail. + if n == 0 { + *ss = 0 + return ssStarter + } + return ssSuccess +} + +// backwards is used for checking for overflow and segment starts +// when traversing a string backwards. Users do not need to call first +// for the first rune. The state of the streamSafe retains the count of +// the non-starters loaded. +func (ss *streamSafe) backwards(p Properties) ssState { + if *ss > maxNonStarters { + panic("streamSafe was not reset") + } + c := *ss + streamSafe(p.nTrailingNonStarters()) + if c > maxNonStarters { + return ssOverflow + } + *ss = c + if p.nLeadingNonStarters() == 0 { + return ssStarter + } + return ssSuccess +} + +func (ss streamSafe) isMax() bool { + return ss == maxNonStarters +} + +// GraphemeJoiner is inserted after maxNonStarters non-starter runes. +const GraphemeJoiner = "\u034F" + +// reorderBuffer is used to normalize a single segment. Characters inserted with +// insert are decomposed and reordered based on CCC. The compose method can +// be used to recombine characters. Note that the byte buffer does not hold +// the UTF-8 characters in order. Only the rune array is maintained in sorted +// order. flush writes the resulting segment to a byte array. +type reorderBuffer struct { + rune [maxBufferSize]Properties // Per character info. + byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos. + nbyte uint8 // Number or bytes. + ss streamSafe // For limiting length of non-starter sequence. + nrune int // Number of runeInfos. + f formInfo + + src input + nsrc int + tmpBytes input + + out []byte + flushF func(*reorderBuffer) bool +} + +func (rb *reorderBuffer) init(f Form, src []byte) { + rb.f = *formTable[f] + rb.src.setBytes(src) + rb.nsrc = len(src) + rb.ss = 0 +} + +func (rb *reorderBuffer) initString(f Form, src string) { + rb.f = *formTable[f] + rb.src.setString(src) + rb.nsrc = len(src) + rb.ss = 0 +} + +func (rb *reorderBuffer) setFlusher(out []byte, f func(*reorderBuffer) bool) { + rb.out = out + rb.flushF = f +} + +// reset discards all characters from the buffer. +func (rb *reorderBuffer) reset() { + rb.nrune = 0 + rb.nbyte = 0 + rb.ss = 0 +} + +func (rb *reorderBuffer) doFlush() bool { + if rb.f.composing { + rb.compose() + } + res := rb.flushF(rb) + rb.reset() + return res +} + +// appendFlush appends the normalized segment to rb.out. +func appendFlush(rb *reorderBuffer) bool { + for i := 0; i < rb.nrune; i++ { + start := rb.rune[i].pos + end := start + rb.rune[i].size + rb.out = append(rb.out, rb.byte[start:end]...) + } + return true +} + +// flush appends the normalized segment to out and resets rb. +func (rb *reorderBuffer) flush(out []byte) []byte { + for i := 0; i < rb.nrune; i++ { + start := rb.rune[i].pos + end := start + rb.rune[i].size + out = append(out, rb.byte[start:end]...) + } + rb.reset() + return out +} + +// flushCopy copies the normalized segment to buf and resets rb. +// It returns the number of bytes written to buf. +func (rb *reorderBuffer) flushCopy(buf []byte) int { + p := 0 + for i := 0; i < rb.nrune; i++ { + runep := rb.rune[i] + p += copy(buf[p:], rb.byte[runep.pos:runep.pos+runep.size]) + } + rb.reset() + return p +} + +// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class. +// It returns false if the buffer is not large enough to hold the rune. +// It is used internally by insert and insertString only. +func (rb *reorderBuffer) insertOrdered(info Properties) { + n := rb.nrune + b := rb.rune[:] + cc := info.ccc + if cc > 0 { + // Find insertion position + move elements to make room. + for ; n > 0; n-- { + if b[n-1].ccc <= cc { + break + } + b[n] = b[n-1] + } + } + rb.nrune += 1 + pos := uint8(rb.nbyte) + rb.nbyte += utf8.UTFMax + info.pos = pos + b[n] = info +} + +// insertErr is an error code returned by insert. Using this type instead +// of error improves performance up to 20% for many of the benchmarks. +type insertErr int + +const ( + iSuccess insertErr = -iota + iShortDst + iShortSrc +) + +// insertFlush inserts the given rune in the buffer ordered by CCC. +// If a decomposition with multiple segments are encountered, they leading +// ones are flushed. +// It returns a non-zero error code if the rune was not inserted. +func (rb *reorderBuffer) insertFlush(src input, i int, info Properties) insertErr { + if rune := src.hangul(i); rune != 0 { + rb.decomposeHangul(rune) + return iSuccess + } + if info.hasDecomposition() { + return rb.insertDecomposed(info.Decomposition()) + } + rb.insertSingle(src, i, info) + return iSuccess +} + +// insertUnsafe inserts the given rune in the buffer ordered by CCC. +// It is assumed there is sufficient space to hold the runes. It is the +// responsibility of the caller to ensure this. This can be done by checking +// the state returned by the streamSafe type. +func (rb *reorderBuffer) insertUnsafe(src input, i int, info Properties) { + if rune := src.hangul(i); rune != 0 { + rb.decomposeHangul(rune) + } + if info.hasDecomposition() { + // TODO: inline. + rb.insertDecomposed(info.Decomposition()) + } else { + rb.insertSingle(src, i, info) + } +} + +// insertDecomposed inserts an entry in to the reorderBuffer for each rune +// in dcomp. dcomp must be a sequence of decomposed UTF-8-encoded runes. +// It flushes the buffer on each new segment start. +func (rb *reorderBuffer) insertDecomposed(dcomp []byte) insertErr { + rb.tmpBytes.setBytes(dcomp) + for i := 0; i < len(dcomp); { + info := rb.f.info(rb.tmpBytes, i) + if info.BoundaryBefore() && rb.nrune > 0 && !rb.doFlush() { + return iShortDst + } + i += copy(rb.byte[rb.nbyte:], dcomp[i:i+int(info.size)]) + rb.insertOrdered(info) + } + return iSuccess +} + +// insertSingle inserts an entry in the reorderBuffer for the rune at +// position i. info is the runeInfo for the rune at position i. +func (rb *reorderBuffer) insertSingle(src input, i int, info Properties) { + src.copySlice(rb.byte[rb.nbyte:], i, i+int(info.size)) + rb.insertOrdered(info) +} + +// insertCGJ inserts a Combining Grapheme Joiner (0x034f) into rb. +func (rb *reorderBuffer) insertCGJ() { + rb.insertSingle(input{str: GraphemeJoiner}, 0, Properties{size: uint8(len(GraphemeJoiner))}) +} + +// appendRune inserts a rune at the end of the buffer. It is used for Hangul. +func (rb *reorderBuffer) appendRune(r rune) { + bn := rb.nbyte + sz := utf8.EncodeRune(rb.byte[bn:], rune(r)) + rb.nbyte += utf8.UTFMax + rb.rune[rb.nrune] = Properties{pos: bn, size: uint8(sz)} + rb.nrune++ +} + +// assignRune sets a rune at position pos. It is used for Hangul and recomposition. +func (rb *reorderBuffer) assignRune(pos int, r rune) { + bn := rb.rune[pos].pos + sz := utf8.EncodeRune(rb.byte[bn:], rune(r)) + rb.rune[pos] = Properties{pos: bn, size: uint8(sz)} +} + +// runeAt returns the rune at position n. It is used for Hangul and recomposition. +func (rb *reorderBuffer) runeAt(n int) rune { + inf := rb.rune[n] + r, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size]) + return r +} + +// bytesAt returns the UTF-8 encoding of the rune at position n. +// It is used for Hangul and recomposition. +func (rb *reorderBuffer) bytesAt(n int) []byte { + inf := rb.rune[n] + return rb.byte[inf.pos : int(inf.pos)+int(inf.size)] +} + +// For Hangul we combine algorithmically, instead of using tables. +const ( + hangulBase = 0xAC00 // UTF-8(hangulBase) -> EA B0 80 + hangulBase0 = 0xEA + hangulBase1 = 0xB0 + hangulBase2 = 0x80 + + hangulEnd = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4 + hangulEnd0 = 0xED + hangulEnd1 = 0x9E + hangulEnd2 = 0xA4 + + jamoLBase = 0x1100 // UTF-8(jamoLBase) -> E1 84 00 + jamoLBase0 = 0xE1 + jamoLBase1 = 0x84 + jamoLEnd = 0x1113 + jamoVBase = 0x1161 + jamoVEnd = 0x1176 + jamoTBase = 0x11A7 + jamoTEnd = 0x11C3 + + jamoTCount = 28 + jamoVCount = 21 + jamoVTCount = 21 * 28 + jamoLVTCount = 19 * 21 * 28 +) + +const hangulUTF8Size = 3 + +func isHangul(b []byte) bool { + if len(b) < hangulUTF8Size { + return false + } + b0 := b[0] + if b0 < hangulBase0 { + return false + } + b1 := b[1] + switch { + case b0 == hangulBase0: + return b1 >= hangulBase1 + case b0 < hangulEnd0: + return true + case b0 > hangulEnd0: + return false + case b1 < hangulEnd1: + return true + } + return b1 == hangulEnd1 && b[2] < hangulEnd2 +} + +func isHangulString(b string) bool { + if len(b) < hangulUTF8Size { + return false + } + b0 := b[0] + if b0 < hangulBase0 { + return false + } + b1 := b[1] + switch { + case b0 == hangulBase0: + return b1 >= hangulBase1 + case b0 < hangulEnd0: + return true + case b0 > hangulEnd0: + return false + case b1 < hangulEnd1: + return true + } + return b1 == hangulEnd1 && b[2] < hangulEnd2 +} + +// Caller must ensure len(b) >= 2. +func isJamoVT(b []byte) bool { + // True if (rune & 0xff00) == jamoLBase + return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1 +} + +func isHangulWithoutJamoT(b []byte) bool { + c, _ := utf8.DecodeRune(b) + c -= hangulBase + return c < jamoLVTCount && c%jamoTCount == 0 +} + +// decomposeHangul writes the decomposed Hangul to buf and returns the number +// of bytes written. len(buf) should be at least 9. +func decomposeHangul(buf []byte, r rune) int { + const JamoUTF8Len = 3 + r -= hangulBase + x := r % jamoTCount + r /= jamoTCount + utf8.EncodeRune(buf, jamoLBase+r/jamoVCount) + utf8.EncodeRune(buf[JamoUTF8Len:], jamoVBase+r%jamoVCount) + if x != 0 { + utf8.EncodeRune(buf[2*JamoUTF8Len:], jamoTBase+x) + return 3 * JamoUTF8Len + } + return 2 * JamoUTF8Len +} + +// decomposeHangul algorithmically decomposes a Hangul rune into +// its Jamo components. +// See http://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul. +func (rb *reorderBuffer) decomposeHangul(r rune) { + r -= hangulBase + x := r % jamoTCount + r /= jamoTCount + rb.appendRune(jamoLBase + r/jamoVCount) + rb.appendRune(jamoVBase + r%jamoVCount) + if x != 0 { + rb.appendRune(jamoTBase + x) + } +} + +// combineHangul algorithmically combines Jamo character components into Hangul. +// See http://unicode.org/reports/tr15/#Hangul for details on combining Hangul. +func (rb *reorderBuffer) combineHangul(s, i, k int) { + b := rb.rune[:] + bn := rb.nrune + for ; i < bn; i++ { + cccB := b[k-1].ccc + cccC := b[i].ccc + if cccB == 0 { + s = k - 1 + } + if s != k-1 && cccB >= cccC { + // b[i] is blocked by greater-equal cccX below it + b[k] = b[i] + k++ + } else { + l := rb.runeAt(s) // also used to compare to hangulBase + v := rb.runeAt(i) // also used to compare to jamoT + switch { + case jamoLBase <= l && l < jamoLEnd && + jamoVBase <= v && v < jamoVEnd: + // 11xx plus 116x to LV + rb.assignRune(s, hangulBase+ + (l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount) + case hangulBase <= l && l < hangulEnd && + jamoTBase < v && v < jamoTEnd && + ((l-hangulBase)%jamoTCount) == 0: + // ACxx plus 11Ax to LVT + rb.assignRune(s, l+v-jamoTBase) + default: + b[k] = b[i] + k++ + } + } + } + rb.nrune = k +} + +// compose recombines the runes in the buffer. +// It should only be used to recompose a single segment, as it will not +// handle alternations between Hangul and non-Hangul characters correctly. +func (rb *reorderBuffer) compose() { + // UAX #15, section X5 , including Corrigendum #5 + // "In any character sequence beginning with starter S, a character C is + // blocked from S if and only if there is some character B between S + // and C, and either B is a starter or it has the same or higher + // combining class as C." + bn := rb.nrune + if bn == 0 { + return + } + k := 1 + b := rb.rune[:] + for s, i := 0, 1; i < bn; i++ { + if isJamoVT(rb.bytesAt(i)) { + // Redo from start in Hangul mode. Necessary to support + // U+320E..U+321E in NFKC mode. + rb.combineHangul(s, i, k) + return + } + ii := b[i] + // We can only use combineForward as a filter if we later + // get the info for the combined character. This is more + // expensive than using the filter. Using combinesBackward() + // is safe. + if ii.combinesBackward() { + cccB := b[k-1].ccc + cccC := ii.ccc + blocked := false // b[i] blocked by starter or greater or equal CCC? + if cccB == 0 { + s = k - 1 + } else { + blocked = s != k-1 && cccB >= cccC + } + if !blocked { + combined := combine(rb.runeAt(s), rb.runeAt(i)) + if combined != 0 { + rb.assignRune(s, combined) + continue + } + } + } + b[k] = b[i] + k++ + } + rb.nrune = k +} diff --git a/vendor/golang.org/x/text/unicode/norm/forminfo.go b/vendor/golang.org/x/text/unicode/norm/forminfo.go new file mode 100644 index 00000000000..15a67c653a3 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/forminfo.go @@ -0,0 +1,256 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +// This file contains Form-specific logic and wrappers for data in tables.go. + +// Rune info is stored in a separate trie per composing form. A composing form +// and its corresponding decomposing form share the same trie. Each trie maps +// a rune to a uint16. The values take two forms. For v >= 0x8000: +// bits +// 15: 1 (inverse of NFD_QD bit of qcInfo) +// 13..7: qcInfo (see below). isYesD is always true (no decompostion). +// 6..0: ccc (compressed CCC value). +// For v < 0x8000, the respective rune has a decomposition and v is an index +// into a byte array of UTF-8 decomposition sequences and additional info and +// has the form: +//
* [ []] +// The header contains the number of bytes in the decomposition (excluding this +// length byte). The two most significant bits of this length byte correspond +// to bit 5 and 4 of qcInfo (see below). The byte sequence itself starts at v+1. +// The byte sequence is followed by a trailing and leading CCC if the values +// for these are not zero. The value of v determines which ccc are appended +// to the sequences. For v < firstCCC, there are none, for v >= firstCCC, +// the sequence is followed by a trailing ccc, and for v >= firstLeadingCC +// there is an additional leading ccc. The value of tccc itself is the +// trailing CCC shifted left 2 bits. The two least-significant bits of tccc +// are the number of trailing non-starters. + +const ( + qcInfoMask = 0x3F // to clear all but the relevant bits in a qcInfo + headerLenMask = 0x3F // extract the length value from the header byte + headerFlagsMask = 0xC0 // extract the qcInfo bits from the header byte +) + +// Properties provides access to normalization properties of a rune. +type Properties struct { + pos uint8 // start position in reorderBuffer; used in composition.go + size uint8 // length of UTF-8 encoding of this rune + ccc uint8 // leading canonical combining class (ccc if not decomposition) + tccc uint8 // trailing canonical combining class (ccc if not decomposition) + nLead uint8 // number of leading non-starters. + flags qcInfo // quick check flags + index uint16 +} + +// functions dispatchable per form +type lookupFunc func(b input, i int) Properties + +// formInfo holds Form-specific functions and tables. +type formInfo struct { + form Form + composing, compatibility bool // form type + info lookupFunc + nextMain iterFunc +} + +var formTable []*formInfo + +func init() { + formTable = make([]*formInfo, 4) + + for i := range formTable { + f := &formInfo{} + formTable[i] = f + f.form = Form(i) + if Form(i) == NFKD || Form(i) == NFKC { + f.compatibility = true + f.info = lookupInfoNFKC + } else { + f.info = lookupInfoNFC + } + f.nextMain = nextDecomposed + if Form(i) == NFC || Form(i) == NFKC { + f.nextMain = nextComposed + f.composing = true + } + } +} + +// We do not distinguish between boundaries for NFC, NFD, etc. to avoid +// unexpected behavior for the user. For example, in NFD, there is a boundary +// after 'a'. However, 'a' might combine with modifiers, so from the application's +// perspective it is not a good boundary. We will therefore always use the +// boundaries for the combining variants. + +// BoundaryBefore returns true if this rune starts a new segment and +// cannot combine with any rune on the left. +func (p Properties) BoundaryBefore() bool { + if p.ccc == 0 && !p.combinesBackward() { + return true + } + // We assume that the CCC of the first character in a decomposition + // is always non-zero if different from info.ccc and that we can return + // false at this point. This is verified by maketables. + return false +} + +// BoundaryAfter returns true if runes cannot combine with or otherwise +// interact with this or previous runes. +func (p Properties) BoundaryAfter() bool { + // TODO: loosen these conditions. + return p.isInert() +} + +// We pack quick check data in 4 bits: +// 5: Combines forward (0 == false, 1 == true) +// 4..3: NFC_QC Yes(00), No (10), or Maybe (11) +// 2: NFD_QC Yes (0) or No (1). No also means there is a decomposition. +// 1..0: Number of trailing non-starters. +// +// When all 4 bits are zero, the character is inert, meaning it is never +// influenced by normalization. +type qcInfo uint8 + +func (p Properties) isYesC() bool { return p.flags&0x10 == 0 } +func (p Properties) isYesD() bool { return p.flags&0x4 == 0 } + +func (p Properties) combinesForward() bool { return p.flags&0x20 != 0 } +func (p Properties) combinesBackward() bool { return p.flags&0x8 != 0 } // == isMaybe +func (p Properties) hasDecomposition() bool { return p.flags&0x4 != 0 } // == isNoD + +func (p Properties) isInert() bool { + return p.flags&qcInfoMask == 0 && p.ccc == 0 +} + +func (p Properties) multiSegment() bool { + return p.index >= firstMulti && p.index < endMulti +} + +func (p Properties) nLeadingNonStarters() uint8 { + return p.nLead +} + +func (p Properties) nTrailingNonStarters() uint8 { + return uint8(p.flags & 0x03) +} + +// Decomposition returns the decomposition for the underlying rune +// or nil if there is none. +func (p Properties) Decomposition() []byte { + // TODO: create the decomposition for Hangul? + if p.index == 0 { + return nil + } + i := p.index + n := decomps[i] & headerLenMask + i++ + return decomps[i : i+uint16(n)] +} + +// Size returns the length of UTF-8 encoding of the rune. +func (p Properties) Size() int { + return int(p.size) +} + +// CCC returns the canonical combining class of the underlying rune. +func (p Properties) CCC() uint8 { + if p.index >= firstCCCZeroExcept { + return 0 + } + return ccc[p.ccc] +} + +// LeadCCC returns the CCC of the first rune in the decomposition. +// If there is no decomposition, LeadCCC equals CCC. +func (p Properties) LeadCCC() uint8 { + return ccc[p.ccc] +} + +// TrailCCC returns the CCC of the last rune in the decomposition. +// If there is no decomposition, TrailCCC equals CCC. +func (p Properties) TrailCCC() uint8 { + return ccc[p.tccc] +} + +// Recomposition +// We use 32-bit keys instead of 64-bit for the two codepoint keys. +// This clips off the bits of three entries, but we know this will not +// result in a collision. In the unlikely event that changes to +// UnicodeData.txt introduce collisions, the compiler will catch it. +// Note that the recomposition map for NFC and NFKC are identical. + +// combine returns the combined rune or 0 if it doesn't exist. +func combine(a, b rune) rune { + key := uint32(uint16(a))<<16 + uint32(uint16(b)) + return recompMap[key] +} + +func lookupInfoNFC(b input, i int) Properties { + v, sz := b.charinfoNFC(i) + return compInfo(v, sz) +} + +func lookupInfoNFKC(b input, i int) Properties { + v, sz := b.charinfoNFKC(i) + return compInfo(v, sz) +} + +// Properties returns properties for the first rune in s. +func (f Form) Properties(s []byte) Properties { + if f == NFC || f == NFD { + return compInfo(nfcData.lookup(s)) + } + return compInfo(nfkcData.lookup(s)) +} + +// PropertiesString returns properties for the first rune in s. +func (f Form) PropertiesString(s string) Properties { + if f == NFC || f == NFD { + return compInfo(nfcData.lookupString(s)) + } + return compInfo(nfkcData.lookupString(s)) +} + +// compInfo converts the information contained in v and sz +// to a Properties. See the comment at the top of the file +// for more information on the format. +func compInfo(v uint16, sz int) Properties { + if v == 0 { + return Properties{size: uint8(sz)} + } else if v >= 0x8000 { + p := Properties{ + size: uint8(sz), + ccc: uint8(v), + tccc: uint8(v), + flags: qcInfo(v >> 8), + } + if p.ccc > 0 || p.combinesBackward() { + p.nLead = uint8(p.flags & 0x3) + } + return p + } + // has decomposition + h := decomps[v] + f := (qcInfo(h&headerFlagsMask) >> 2) | 0x4 + p := Properties{size: uint8(sz), flags: f, index: v} + if v >= firstCCC { + v += uint16(h&headerLenMask) + 1 + c := decomps[v] + p.tccc = c >> 2 + p.flags |= qcInfo(c & 0x3) + if v >= firstLeadingCCC { + p.nLead = c & 0x3 + if v >= firstStarterWithNLead { + // We were tricked. Remove the decomposition. + p.flags &= 0x03 + p.index = 0 + return p + } + p.ccc = decomps[v+1] + } + } + return p +} diff --git a/vendor/golang.org/x/text/unicode/norm/input.go b/vendor/golang.org/x/text/unicode/norm/input.go new file mode 100644 index 00000000000..045d4ccce2e --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/input.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import "unicode/utf8" + +type input struct { + str string + bytes []byte +} + +func inputBytes(str []byte) input { + return input{bytes: str} +} + +func inputString(str string) input { + return input{str: str} +} + +func (in *input) setBytes(str []byte) { + in.str = "" + in.bytes = str +} + +func (in *input) setString(str string) { + in.str = str + in.bytes = nil +} + +func (in *input) _byte(p int) byte { + if in.bytes == nil { + return in.str[p] + } + return in.bytes[p] +} + +func (in *input) skipASCII(p, max int) int { + if in.bytes == nil { + for ; p < max && in.str[p] < utf8.RuneSelf; p++ { + } + } else { + for ; p < max && in.bytes[p] < utf8.RuneSelf; p++ { + } + } + return p +} + +func (in *input) skipContinuationBytes(p int) int { + if in.bytes == nil { + for ; p < len(in.str) && !utf8.RuneStart(in.str[p]); p++ { + } + } else { + for ; p < len(in.bytes) && !utf8.RuneStart(in.bytes[p]); p++ { + } + } + return p +} + +func (in *input) appendSlice(buf []byte, b, e int) []byte { + if in.bytes != nil { + return append(buf, in.bytes[b:e]...) + } + for i := b; i < e; i++ { + buf = append(buf, in.str[i]) + } + return buf +} + +func (in *input) copySlice(buf []byte, b, e int) int { + if in.bytes == nil { + return copy(buf, in.str[b:e]) + } + return copy(buf, in.bytes[b:e]) +} + +func (in *input) charinfoNFC(p int) (uint16, int) { + if in.bytes == nil { + return nfcData.lookupString(in.str[p:]) + } + return nfcData.lookup(in.bytes[p:]) +} + +func (in *input) charinfoNFKC(p int) (uint16, int) { + if in.bytes == nil { + return nfkcData.lookupString(in.str[p:]) + } + return nfkcData.lookup(in.bytes[p:]) +} + +func (in *input) hangul(p int) (r rune) { + if in.bytes == nil { + if !isHangulString(in.str[p:]) { + return 0 + } + r, _ = utf8.DecodeRuneInString(in.str[p:]) + } else { + if !isHangul(in.bytes[p:]) { + return 0 + } + r, _ = utf8.DecodeRune(in.bytes[p:]) + } + return r +} diff --git a/vendor/golang.org/x/text/unicode/norm/iter.go b/vendor/golang.org/x/text/unicode/norm/iter.go new file mode 100644 index 00000000000..0a42a72de81 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/iter.go @@ -0,0 +1,450 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import ( + "fmt" + "unicode/utf8" +) + +// MaxSegmentSize is the maximum size of a byte buffer needed to consider any +// sequence of starter and non-starter runes for the purpose of normalization. +const MaxSegmentSize = maxByteBufferSize + +// An Iter iterates over a string or byte slice, while normalizing it +// to a given Form. +type Iter struct { + rb reorderBuffer + buf [maxByteBufferSize]byte + info Properties // first character saved from previous iteration + next iterFunc // implementation of next depends on form + asciiF iterFunc + + p int // current position in input source + multiSeg []byte // remainder of multi-segment decomposition +} + +type iterFunc func(*Iter) []byte + +// Init initializes i to iterate over src after normalizing it to Form f. +func (i *Iter) Init(f Form, src []byte) { + i.p = 0 + if len(src) == 0 { + i.setDone() + i.rb.nsrc = 0 + return + } + i.multiSeg = nil + i.rb.init(f, src) + i.next = i.rb.f.nextMain + i.asciiF = nextASCIIBytes + i.info = i.rb.f.info(i.rb.src, i.p) +} + +// InitString initializes i to iterate over src after normalizing it to Form f. +func (i *Iter) InitString(f Form, src string) { + i.p = 0 + if len(src) == 0 { + i.setDone() + i.rb.nsrc = 0 + return + } + i.multiSeg = nil + i.rb.initString(f, src) + i.next = i.rb.f.nextMain + i.asciiF = nextASCIIString + i.info = i.rb.f.info(i.rb.src, i.p) +} + +// Seek sets the segment to be returned by the next call to Next to start +// at position p. It is the responsibility of the caller to set p to the +// start of a UTF8 rune. +func (i *Iter) Seek(offset int64, whence int) (int64, error) { + var abs int64 + switch whence { + case 0: + abs = offset + case 1: + abs = int64(i.p) + offset + case 2: + abs = int64(i.rb.nsrc) + offset + default: + return 0, fmt.Errorf("norm: invalid whence") + } + if abs < 0 { + return 0, fmt.Errorf("norm: negative position") + } + if int(abs) >= i.rb.nsrc { + i.setDone() + return int64(i.p), nil + } + i.p = int(abs) + i.multiSeg = nil + i.next = i.rb.f.nextMain + i.info = i.rb.f.info(i.rb.src, i.p) + return abs, nil +} + +// returnSlice returns a slice of the underlying input type as a byte slice. +// If the underlying is of type []byte, it will simply return a slice. +// If the underlying is of type string, it will copy the slice to the buffer +// and return that. +func (i *Iter) returnSlice(a, b int) []byte { + if i.rb.src.bytes == nil { + return i.buf[:copy(i.buf[:], i.rb.src.str[a:b])] + } + return i.rb.src.bytes[a:b] +} + +// Pos returns the byte position at which the next call to Next will commence processing. +func (i *Iter) Pos() int { + return i.p +} + +func (i *Iter) setDone() { + i.next = nextDone + i.p = i.rb.nsrc +} + +// Done returns true if there is no more input to process. +func (i *Iter) Done() bool { + return i.p >= i.rb.nsrc +} + +// Next returns f(i.input[i.Pos():n]), where n is a boundary of i.input. +// For any input a and b for which f(a) == f(b), subsequent calls +// to Next will return the same segments. +// Modifying runes are grouped together with the preceding starter, if such a starter exists. +// Although not guaranteed, n will typically be the smallest possible n. +func (i *Iter) Next() []byte { + return i.next(i) +} + +func nextASCIIBytes(i *Iter) []byte { + p := i.p + 1 + if p >= i.rb.nsrc { + i.setDone() + return i.rb.src.bytes[i.p:p] + } + if i.rb.src.bytes[p] < utf8.RuneSelf { + p0 := i.p + i.p = p + return i.rb.src.bytes[p0:p] + } + i.info = i.rb.f.info(i.rb.src, i.p) + i.next = i.rb.f.nextMain + return i.next(i) +} + +func nextASCIIString(i *Iter) []byte { + p := i.p + 1 + if p >= i.rb.nsrc { + i.buf[0] = i.rb.src.str[i.p] + i.setDone() + return i.buf[:1] + } + if i.rb.src.str[p] < utf8.RuneSelf { + i.buf[0] = i.rb.src.str[i.p] + i.p = p + return i.buf[:1] + } + i.info = i.rb.f.info(i.rb.src, i.p) + i.next = i.rb.f.nextMain + return i.next(i) +} + +func nextHangul(i *Iter) []byte { + p := i.p + next := p + hangulUTF8Size + if next >= i.rb.nsrc { + i.setDone() + } else if i.rb.src.hangul(next) == 0 { + i.info = i.rb.f.info(i.rb.src, i.p) + i.next = i.rb.f.nextMain + return i.next(i) + } + i.p = next + return i.buf[:decomposeHangul(i.buf[:], i.rb.src.hangul(p))] +} + +func nextDone(i *Iter) []byte { + return nil +} + +// nextMulti is used for iterating over multi-segment decompositions +// for decomposing normal forms. +func nextMulti(i *Iter) []byte { + j := 0 + d := i.multiSeg + // skip first rune + for j = 1; j < len(d) && !utf8.RuneStart(d[j]); j++ { + } + for j < len(d) { + info := i.rb.f.info(input{bytes: d}, j) + if info.BoundaryBefore() { + i.multiSeg = d[j:] + return d[:j] + } + j += int(info.size) + } + // treat last segment as normal decomposition + i.next = i.rb.f.nextMain + return i.next(i) +} + +// nextMultiNorm is used for iterating over multi-segment decompositions +// for composing normal forms. +func nextMultiNorm(i *Iter) []byte { + j := 0 + d := i.multiSeg + for j < len(d) { + info := i.rb.f.info(input{bytes: d}, j) + if info.BoundaryBefore() { + i.rb.compose() + seg := i.buf[:i.rb.flushCopy(i.buf[:])] + i.rb.ss.first(info) + i.rb.insertUnsafe(input{bytes: d}, j, info) + i.multiSeg = d[j+int(info.size):] + return seg + } + i.rb.ss.next(info) + i.rb.insertUnsafe(input{bytes: d}, j, info) + j += int(info.size) + } + i.multiSeg = nil + i.next = nextComposed + return doNormComposed(i) +} + +// nextDecomposed is the implementation of Next for forms NFD and NFKD. +func nextDecomposed(i *Iter) (next []byte) { + outp := 0 + inCopyStart, outCopyStart := i.p, 0 + ss := mkStreamSafe(i.info) + for { + if sz := int(i.info.size); sz <= 1 { + p := i.p + i.p++ // ASCII or illegal byte. Either way, advance by 1. + if i.p >= i.rb.nsrc { + i.setDone() + return i.returnSlice(p, i.p) + } else if i.rb.src._byte(i.p) < utf8.RuneSelf { + i.next = i.asciiF + return i.returnSlice(p, i.p) + } + outp++ + } else if d := i.info.Decomposition(); d != nil { + // Note: If leading CCC != 0, then len(d) == 2 and last is also non-zero. + // Case 1: there is a leftover to copy. In this case the decomposition + // must begin with a modifier and should always be appended. + // Case 2: no leftover. Simply return d if followed by a ccc == 0 value. + p := outp + len(d) + if outp > 0 { + i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p) + if p > len(i.buf) { + return i.buf[:outp] + } + } else if i.info.multiSegment() { + // outp must be 0 as multi-segment decompositions always + // start a new segment. + if i.multiSeg == nil { + i.multiSeg = d + i.next = nextMulti + return nextMulti(i) + } + // We are in the last segment. Treat as normal decomposition. + d = i.multiSeg + i.multiSeg = nil + p = len(d) + } + prevCC := i.info.tccc + if i.p += sz; i.p >= i.rb.nsrc { + i.setDone() + i.info = Properties{} // Force BoundaryBefore to succeed. + } else { + i.info = i.rb.f.info(i.rb.src, i.p) + } + switch ss.next(i.info) { + case ssOverflow: + i.next = nextCGJDecompose + fallthrough + case ssStarter: + if outp > 0 { + copy(i.buf[outp:], d) + return i.buf[:p] + } + return d + } + copy(i.buf[outp:], d) + outp = p + inCopyStart, outCopyStart = i.p, outp + if i.info.ccc < prevCC { + goto doNorm + } + continue + } else if r := i.rb.src.hangul(i.p); r != 0 { + outp = decomposeHangul(i.buf[:], r) + i.p += hangulUTF8Size + inCopyStart, outCopyStart = i.p, outp + if i.p >= i.rb.nsrc { + i.setDone() + break + } else if i.rb.src.hangul(i.p) != 0 { + i.next = nextHangul + return i.buf[:outp] + } + } else { + p := outp + sz + if p > len(i.buf) { + break + } + outp = p + i.p += sz + } + if i.p >= i.rb.nsrc { + i.setDone() + break + } + prevCC := i.info.tccc + i.info = i.rb.f.info(i.rb.src, i.p) + if v := ss.next(i.info); v == ssStarter { + break + } else if v == ssOverflow { + i.next = nextCGJDecompose + break + } + if i.info.ccc < prevCC { + goto doNorm + } + } + if outCopyStart == 0 { + return i.returnSlice(inCopyStart, i.p) + } else if inCopyStart < i.p { + i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p) + } + return i.buf[:outp] +doNorm: + // Insert what we have decomposed so far in the reorderBuffer. + // As we will only reorder, there will always be enough room. + i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p) + i.rb.insertDecomposed(i.buf[0:outp]) + return doNormDecomposed(i) +} + +func doNormDecomposed(i *Iter) []byte { + for { + if s := i.rb.ss.next(i.info); s == ssOverflow { + i.next = nextCGJDecompose + break + } + i.rb.insertUnsafe(i.rb.src, i.p, i.info) + if i.p += int(i.info.size); i.p >= i.rb.nsrc { + i.setDone() + break + } + i.info = i.rb.f.info(i.rb.src, i.p) + if i.info.ccc == 0 { + break + } + } + // new segment or too many combining characters: exit normalization + return i.buf[:i.rb.flushCopy(i.buf[:])] +} + +func nextCGJDecompose(i *Iter) []byte { + i.rb.ss = 0 + i.rb.insertCGJ() + i.next = nextDecomposed + buf := doNormDecomposed(i) + return buf +} + +// nextComposed is the implementation of Next for forms NFC and NFKC. +func nextComposed(i *Iter) []byte { + outp, startp := 0, i.p + var prevCC uint8 + ss := mkStreamSafe(i.info) + for { + if !i.info.isYesC() { + goto doNorm + } + prevCC = i.info.tccc + sz := int(i.info.size) + if sz == 0 { + sz = 1 // illegal rune: copy byte-by-byte + } + p := outp + sz + if p > len(i.buf) { + break + } + outp = p + i.p += sz + if i.p >= i.rb.nsrc { + i.setDone() + break + } else if i.rb.src._byte(i.p) < utf8.RuneSelf { + i.next = i.asciiF + break + } + i.info = i.rb.f.info(i.rb.src, i.p) + if v := ss.next(i.info); v == ssStarter { + break + } else if v == ssOverflow { + i.next = nextCGJCompose + break + } + if i.info.ccc < prevCC { + goto doNorm + } + } + return i.returnSlice(startp, i.p) +doNorm: + i.p = startp + i.info = i.rb.f.info(i.rb.src, i.p) + if i.info.multiSegment() { + d := i.info.Decomposition() + info := i.rb.f.info(input{bytes: d}, 0) + i.rb.insertUnsafe(input{bytes: d}, 0, info) + i.multiSeg = d[int(info.size):] + i.next = nextMultiNorm + return nextMultiNorm(i) + } + i.rb.ss.first(i.info) + i.rb.insertUnsafe(i.rb.src, i.p, i.info) + return doNormComposed(i) +} + +func doNormComposed(i *Iter) []byte { + // First rune should already be inserted. + for { + if i.p += int(i.info.size); i.p >= i.rb.nsrc { + i.setDone() + break + } + i.info = i.rb.f.info(i.rb.src, i.p) + if s := i.rb.ss.next(i.info); s == ssStarter { + break + } else if s == ssOverflow { + i.next = nextCGJCompose + break + } + i.rb.insertUnsafe(i.rb.src, i.p, i.info) + } + i.rb.compose() + seg := i.buf[:i.rb.flushCopy(i.buf[:])] + return seg +} + +func nextCGJCompose(i *Iter) []byte { + i.rb.ss = 0 // instead of first + i.rb.insertCGJ() + i.next = nextComposed + // Note that we treat any rune with nLeadingNonStarters > 0 as a non-starter, + // even if they are not. This is particularly dubious for U+FF9E and UFF9A. + // If we ever change that, insert a check here. + i.rb.ss.first(i.info) + i.rb.insertUnsafe(i.rb.src, i.p, i.info) + return doNormComposed(i) +} diff --git a/vendor/golang.org/x/text/unicode/norm/maketables.go b/vendor/golang.org/x/text/unicode/norm/maketables.go new file mode 100644 index 00000000000..07bdff6bdcd --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/maketables.go @@ -0,0 +1,978 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Normalization table generator. +// Data read from the web. +// See forminfo.go for a description of the trie values associated with each rune. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io" + "log" + "sort" + "strconv" + "strings" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/triegen" + "golang.org/x/text/internal/ucd" +) + +func main() { + gen.Init() + loadUnicodeData() + compactCCC() + loadCompositionExclusions() + completeCharFields(FCanonical) + completeCharFields(FCompatibility) + computeNonStarterCounts() + verifyComputed() + printChars() + if *test { + testDerived() + printTestdata() + } else { + makeTables() + } +} + +var ( + tablelist = flag.String("tables", + "all", + "comma-separated list of which tables to generate; "+ + "can be 'decomp', 'recomp', 'info' and 'all'") + test = flag.Bool("test", + false, + "test existing tables against DerivedNormalizationProps and generate test data for regression testing") + verbose = flag.Bool("verbose", + false, + "write data to stdout as it is parsed") +) + +const MaxChar = 0x10FFFF // anything above this shouldn't exist + +// Quick Check properties of runes allow us to quickly +// determine whether a rune may occur in a normal form. +// For a given normal form, a rune may be guaranteed to occur +// verbatim (QC=Yes), may or may not combine with another +// rune (QC=Maybe), or may not occur (QC=No). +type QCResult int + +const ( + QCUnknown QCResult = iota + QCYes + QCNo + QCMaybe +) + +func (r QCResult) String() string { + switch r { + case QCYes: + return "Yes" + case QCNo: + return "No" + case QCMaybe: + return "Maybe" + } + return "***UNKNOWN***" +} + +const ( + FCanonical = iota // NFC or NFD + FCompatibility // NFKC or NFKD + FNumberOfFormTypes +) + +const ( + MComposed = iota // NFC or NFKC + MDecomposed // NFD or NFKD + MNumberOfModes +) + +// This contains only the properties we're interested in. +type Char struct { + name string + codePoint rune // if zero, this index is not a valid code point. + ccc uint8 // canonical combining class + origCCC uint8 + excludeInComp bool // from CompositionExclusions.txt + compatDecomp bool // it has a compatibility expansion + + nTrailingNonStarters uint8 + nLeadingNonStarters uint8 // must be equal to trailing if non-zero + + forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility + + state State +} + +var chars = make([]Char, MaxChar+1) +var cccMap = make(map[uint8]uint8) + +func (c Char) String() string { + buf := new(bytes.Buffer) + + fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name) + fmt.Fprintf(buf, " ccc: %v\n", c.ccc) + fmt.Fprintf(buf, " excludeInComp: %v\n", c.excludeInComp) + fmt.Fprintf(buf, " compatDecomp: %v\n", c.compatDecomp) + fmt.Fprintf(buf, " state: %v\n", c.state) + fmt.Fprintf(buf, " NFC:\n") + fmt.Fprint(buf, c.forms[FCanonical]) + fmt.Fprintf(buf, " NFKC:\n") + fmt.Fprint(buf, c.forms[FCompatibility]) + + return buf.String() +} + +// In UnicodeData.txt, some ranges are marked like this: +// 3400;;Lo;0;L;;;;;N;;;;; +// 4DB5;;Lo;0;L;;;;;N;;;;; +// parseCharacter keeps a state variable indicating the weirdness. +type State int + +const ( + SNormal State = iota // known to be zero for the type + SFirst + SLast + SMissing +) + +var lastChar = rune('\u0000') + +func (c Char) isValid() bool { + return c.codePoint != 0 && c.state != SMissing +} + +type FormInfo struct { + quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed + verified [MNumberOfModes]bool // index: MComposed or MDecomposed + + combinesForward bool // May combine with rune on the right + combinesBackward bool // May combine with rune on the left + isOneWay bool // Never appears in result + inDecomp bool // Some decompositions result in this char. + decomp Decomposition + expandedDecomp Decomposition +} + +func (f FormInfo) String() string { + buf := bytes.NewBuffer(make([]byte, 0)) + + fmt.Fprintf(buf, " quickCheck[C]: %v\n", f.quickCheck[MComposed]) + fmt.Fprintf(buf, " quickCheck[D]: %v\n", f.quickCheck[MDecomposed]) + fmt.Fprintf(buf, " cmbForward: %v\n", f.combinesForward) + fmt.Fprintf(buf, " cmbBackward: %v\n", f.combinesBackward) + fmt.Fprintf(buf, " isOneWay: %v\n", f.isOneWay) + fmt.Fprintf(buf, " inDecomp: %v\n", f.inDecomp) + fmt.Fprintf(buf, " decomposition: %X\n", f.decomp) + fmt.Fprintf(buf, " expandedDecomp: %X\n", f.expandedDecomp) + + return buf.String() +} + +type Decomposition []rune + +func parseDecomposition(s string, skipfirst bool) (a []rune, err error) { + decomp := strings.Split(s, " ") + if len(decomp) > 0 && skipfirst { + decomp = decomp[1:] + } + for _, d := range decomp { + point, err := strconv.ParseUint(d, 16, 64) + if err != nil { + return a, err + } + a = append(a, rune(point)) + } + return a, nil +} + +func loadUnicodeData() { + f := gen.OpenUCDFile("UnicodeData.txt") + defer f.Close() + p := ucd.New(f) + for p.Next() { + r := p.Rune(ucd.CodePoint) + char := &chars[r] + + char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass)) + decmap := p.String(ucd.DecompMapping) + + exp, err := parseDecomposition(decmap, false) + isCompat := false + if err != nil { + if len(decmap) > 0 { + exp, err = parseDecomposition(decmap, true) + if err != nil { + log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err) + } + isCompat = true + } + } + + char.name = p.String(ucd.Name) + char.codePoint = r + char.forms[FCompatibility].decomp = exp + if !isCompat { + char.forms[FCanonical].decomp = exp + } else { + char.compatDecomp = true + } + if len(decmap) > 0 { + char.forms[FCompatibility].decomp = exp + } + } + if err := p.Err(); err != nil { + log.Fatal(err) + } +} + +// compactCCC converts the sparse set of CCC values to a continguous one, +// reducing the number of bits needed from 8 to 6. +func compactCCC() { + m := make(map[uint8]uint8) + for i := range chars { + c := &chars[i] + m[c.ccc] = 0 + } + cccs := []int{} + for v, _ := range m { + cccs = append(cccs, int(v)) + } + sort.Ints(cccs) + for i, c := range cccs { + cccMap[uint8(i)] = uint8(c) + m[uint8(c)] = uint8(i) + } + for i := range chars { + c := &chars[i] + c.origCCC = c.ccc + c.ccc = m[c.ccc] + } + if len(m) >= 1<<6 { + log.Fatalf("too many difference CCC values: %d >= 64", len(m)) + } +} + +// CompositionExclusions.txt has form: +// 0958 # ... +// See http://unicode.org/reports/tr44/ for full explanation +func loadCompositionExclusions() { + f := gen.OpenUCDFile("CompositionExclusions.txt") + defer f.Close() + p := ucd.New(f) + for p.Next() { + c := &chars[p.Rune(0)] + if c.excludeInComp { + log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint) + } + c.excludeInComp = true + } + if e := p.Err(); e != nil { + log.Fatal(e) + } +} + +// hasCompatDecomp returns true if any of the recursive +// decompositions contains a compatibility expansion. +// In this case, the character may not occur in NFK*. +func hasCompatDecomp(r rune) bool { + c := &chars[r] + if c.compatDecomp { + return true + } + for _, d := range c.forms[FCompatibility].decomp { + if hasCompatDecomp(d) { + return true + } + } + return false +} + +// Hangul related constants. +const ( + HangulBase = 0xAC00 + HangulEnd = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28) + + JamoLBase = 0x1100 + JamoLEnd = 0x1113 + JamoVBase = 0x1161 + JamoVEnd = 0x1176 + JamoTBase = 0x11A8 + JamoTEnd = 0x11C3 + + JamoLVTCount = 19 * 21 * 28 + JamoTCount = 28 +) + +func isHangul(r rune) bool { + return HangulBase <= r && r < HangulEnd +} + +func isHangulWithoutJamoT(r rune) bool { + if !isHangul(r) { + return false + } + r -= HangulBase + return r < JamoLVTCount && r%JamoTCount == 0 +} + +func ccc(r rune) uint8 { + return chars[r].ccc +} + +// Insert a rune in a buffer, ordered by Canonical Combining Class. +func insertOrdered(b Decomposition, r rune) Decomposition { + n := len(b) + b = append(b, 0) + cc := ccc(r) + if cc > 0 { + // Use bubble sort. + for ; n > 0; n-- { + if ccc(b[n-1]) <= cc { + break + } + b[n] = b[n-1] + } + } + b[n] = r + return b +} + +// Recursively decompose. +func decomposeRecursive(form int, r rune, d Decomposition) Decomposition { + dcomp := chars[r].forms[form].decomp + if len(dcomp) == 0 { + return insertOrdered(d, r) + } + for _, c := range dcomp { + d = decomposeRecursive(form, c, d) + } + return d +} + +func completeCharFields(form int) { + // Phase 0: pre-expand decomposition. + for i := range chars { + f := &chars[i].forms[form] + if len(f.decomp) == 0 { + continue + } + exp := make(Decomposition, 0) + for _, c := range f.decomp { + exp = decomposeRecursive(form, c, exp) + } + f.expandedDecomp = exp + } + + // Phase 1: composition exclusion, mark decomposition. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + // Marks script-specific exclusions and version restricted. + f.isOneWay = c.excludeInComp + + // Singletons + f.isOneWay = f.isOneWay || len(f.decomp) == 1 + + // Non-starter decompositions + if len(f.decomp) > 1 { + chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0 + f.isOneWay = f.isOneWay || chk + } + + // Runes that decompose into more than two runes. + f.isOneWay = f.isOneWay || len(f.decomp) > 2 + + if form == FCompatibility { + f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint) + } + + for _, r := range f.decomp { + chars[r].forms[form].inDecomp = true + } + } + + // Phase 2: forward and backward combining. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + if !f.isOneWay && len(f.decomp) == 2 { + f0 := &chars[f.decomp[0]].forms[form] + f1 := &chars[f.decomp[1]].forms[form] + if !f0.isOneWay { + f0.combinesForward = true + } + if !f1.isOneWay { + f1.combinesBackward = true + } + } + if isHangulWithoutJamoT(rune(i)) { + f.combinesForward = true + } + } + + // Phase 3: quick check values. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + switch { + case len(f.decomp) > 0: + f.quickCheck[MDecomposed] = QCNo + case isHangul(rune(i)): + f.quickCheck[MDecomposed] = QCNo + default: + f.quickCheck[MDecomposed] = QCYes + } + switch { + case f.isOneWay: + f.quickCheck[MComposed] = QCNo + case (i & 0xffff00) == JamoLBase: + f.quickCheck[MComposed] = QCYes + if JamoLBase <= i && i < JamoLEnd { + f.combinesForward = true + } + if JamoVBase <= i && i < JamoVEnd { + f.quickCheck[MComposed] = QCMaybe + f.combinesBackward = true + f.combinesForward = true + } + if JamoTBase <= i && i < JamoTEnd { + f.quickCheck[MComposed] = QCMaybe + f.combinesBackward = true + } + case !f.combinesBackward: + f.quickCheck[MComposed] = QCYes + default: + f.quickCheck[MComposed] = QCMaybe + } + } +} + +func computeNonStarterCounts() { + // Phase 4: leading and trailing non-starter count + for i := range chars { + c := &chars[i] + + runes := []rune{rune(i)} + // We always use FCompatibility so that the CGJ insertion points do not + // change for repeated normalizations with different forms. + if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 { + runes = exp + } + // We consider runes that combine backwards to be non-starters for the + // purpose of Stream-Safe Text Processing. + for _, r := range runes { + if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { + break + } + c.nLeadingNonStarters++ + } + for i := len(runes) - 1; i >= 0; i-- { + if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { + break + } + c.nTrailingNonStarters++ + } + if c.nTrailingNonStarters > 3 { + log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes) + } + + if isHangul(rune(i)) { + c.nTrailingNonStarters = 2 + if isHangulWithoutJamoT(rune(i)) { + c.nTrailingNonStarters = 1 + } + } + + if l, t := c.nLeadingNonStarters, c.nTrailingNonStarters; l > 0 && l != t { + log.Fatalf("%U: number of leading and trailing non-starters should be equal (%d vs %d)", i, l, t) + } + if t := c.nTrailingNonStarters; t > 3 { + log.Fatalf("%U: number of trailing non-starters is %d > 3", t) + } + } +} + +func printBytes(w io.Writer, b []byte, name string) { + fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b)) + fmt.Fprintf(w, "var %s = [...]byte {", name) + for i, c := range b { + switch { + case i%64 == 0: + fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63) + case i%8 == 0: + fmt.Fprintf(w, "\n") + } + fmt.Fprintf(w, "0x%.2X, ", c) + } + fmt.Fprint(w, "\n}\n\n") +} + +// See forminfo.go for format. +func makeEntry(f *FormInfo, c *Char) uint16 { + e := uint16(0) + if r := c.codePoint; HangulBase <= r && r < HangulEnd { + e |= 0x40 + } + if f.combinesForward { + e |= 0x20 + } + if f.quickCheck[MDecomposed] == QCNo { + e |= 0x4 + } + switch f.quickCheck[MComposed] { + case QCYes: + case QCNo: + e |= 0x10 + case QCMaybe: + e |= 0x18 + default: + log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed]) + } + e |= uint16(c.nTrailingNonStarters) + return e +} + +// decompSet keeps track of unique decompositions, grouped by whether +// the decomposition is followed by a trailing and/or leading CCC. +type decompSet [7]map[string]bool + +const ( + normalDecomp = iota + firstMulti + firstCCC + endMulti + firstLeadingCCC + firstCCCZeroExcept + firstStarterWithNLead + lastDecomp +) + +var cname = []string{"firstMulti", "firstCCC", "endMulti", "firstLeadingCCC", "firstCCCZeroExcept", "firstStarterWithNLead", "lastDecomp"} + +func makeDecompSet() decompSet { + m := decompSet{} + for i := range m { + m[i] = make(map[string]bool) + } + return m +} +func (m *decompSet) insert(key int, s string) { + m[key][s] = true +} + +func printCharInfoTables(w io.Writer) int { + mkstr := func(r rune, f *FormInfo) (int, string) { + d := f.expandedDecomp + s := string([]rune(d)) + if max := 1 << 6; len(s) >= max { + const msg = "%U: too many bytes in decomposition: %d >= %d" + log.Fatalf(msg, r, len(s), max) + } + head := uint8(len(s)) + if f.quickCheck[MComposed] != QCYes { + head |= 0x40 + } + if f.combinesForward { + head |= 0x80 + } + s = string([]byte{head}) + s + + lccc := ccc(d[0]) + tccc := ccc(d[len(d)-1]) + cc := ccc(r) + if cc != 0 && lccc == 0 && tccc == 0 { + log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc) + } + if tccc < lccc && lccc != 0 { + const msg = "%U: lccc (%d) must be <= tcc (%d)" + log.Fatalf(msg, r, lccc, tccc) + } + index := normalDecomp + nTrail := chars[r].nTrailingNonStarters + if tccc > 0 || lccc > 0 || nTrail > 0 { + tccc <<= 2 + tccc |= nTrail + s += string([]byte{tccc}) + index = endMulti + for _, r := range d[1:] { + if ccc(r) == 0 { + index = firstCCC + } + } + if lccc > 0 { + s += string([]byte{lccc}) + if index == firstCCC { + log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r) + } + index = firstLeadingCCC + } + if cc != lccc { + if cc != 0 { + log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc) + } + index = firstCCCZeroExcept + } + } else if len(d) > 1 { + index = firstMulti + } + return index, s + } + + decompSet := makeDecompSet() + const nLeadStr = "\x00\x01" // 0-byte length and tccc with nTrail. + decompSet.insert(firstStarterWithNLead, nLeadStr) + + // Store the uniqued decompositions in a byte buffer, + // preceded by their byte length. + for _, c := range chars { + for _, f := range c.forms { + if len(f.expandedDecomp) == 0 { + continue + } + if f.combinesBackward { + log.Fatalf("%U: combinesBackward and decompose", c.codePoint) + } + index, s := mkstr(c.codePoint, &f) + decompSet.insert(index, s) + } + } + + decompositions := bytes.NewBuffer(make([]byte, 0, 10000)) + size := 0 + positionMap := make(map[string]uint16) + decompositions.WriteString("\000") + fmt.Fprintln(w, "const (") + for i, m := range decompSet { + sa := []string{} + for s := range m { + sa = append(sa, s) + } + sort.Strings(sa) + for _, s := range sa { + p := decompositions.Len() + decompositions.WriteString(s) + positionMap[s] = uint16(p) + } + if cname[i] != "" { + fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len()) + } + } + fmt.Fprintln(w, "maxDecomp = 0x8000") + fmt.Fprintln(w, ")") + b := decompositions.Bytes() + printBytes(w, b, "decomps") + size += len(b) + + varnames := []string{"nfc", "nfkc"} + for i := 0; i < FNumberOfFormTypes; i++ { + trie := triegen.NewTrie(varnames[i]) + + for r, c := range chars { + f := c.forms[i] + d := f.expandedDecomp + if len(d) != 0 { + _, key := mkstr(c.codePoint, &f) + trie.Insert(rune(r), uint64(positionMap[key])) + if c.ccc != ccc(d[0]) { + // We assume the lead ccc of a decomposition !=0 in this case. + if ccc(d[0]) == 0 { + log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc) + } + } + } else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward { + // Handle cases where it can't be detected that the nLead should be equal + // to nTrail. + trie.Insert(c.codePoint, uint64(positionMap[nLeadStr])) + } else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 { + trie.Insert(c.codePoint, uint64(0x8000|v)) + } + } + sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]})) + if err != nil { + log.Fatal(err) + } + size += sz + } + return size +} + +func contains(sa []string, s string) bool { + for _, a := range sa { + if a == s { + return true + } + } + return false +} + +func makeTables() { + w := &bytes.Buffer{} + + size := 0 + if *tablelist == "" { + return + } + list := strings.Split(*tablelist, ",") + if *tablelist == "all" { + list = []string{"recomp", "info"} + } + + // Compute maximum decomposition size. + max := 0 + for _, c := range chars { + if n := len(string(c.forms[FCompatibility].expandedDecomp)); n > max { + max = n + } + } + + fmt.Fprintln(w, "const (") + fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.") + fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion()) + fmt.Fprintln(w) + fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform") + fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at") + fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that") + fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.") + fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max) + fmt.Fprintln(w, ")\n") + + // Print the CCC remap table. + size += len(cccMap) + fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap)) + for i := 0; i < len(cccMap); i++ { + if i%8 == 0 { + fmt.Fprintln(w) + } + fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)]) + } + fmt.Fprintln(w, "\n}\n") + + if contains(list, "info") { + size += printCharInfoTables(w) + } + + if contains(list, "recomp") { + // Note that we use 32 bit keys, instead of 64 bit. + // This clips the bits of three entries, but we know + // this won't cause a collision. The compiler will catch + // any changes made to UnicodeData.txt that introduces + // a collision. + // Note that the recomposition map for NFC and NFKC + // are identical. + + // Recomposition map + nrentries := 0 + for _, c := range chars { + f := c.forms[FCanonical] + if !f.isOneWay && len(f.decomp) > 0 { + nrentries++ + } + } + sz := nrentries * 8 + size += sz + fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz) + fmt.Fprintln(w, "var recompMap = map[uint32]rune{") + for i, c := range chars { + f := c.forms[FCanonical] + d := f.decomp + if !f.isOneWay && len(d) > 0 { + key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1])) + fmt.Fprintf(w, "0x%.8X: 0x%.4X,\n", key, i) + } + } + fmt.Fprintf(w, "}\n\n") + } + + fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size) + gen.WriteGoFile("tables.go", "norm", w.Bytes()) +} + +func printChars() { + if *verbose { + for _, c := range chars { + if !c.isValid() || c.state == SMissing { + continue + } + fmt.Println(c) + } + } +} + +// verifyComputed does various consistency tests. +func verifyComputed() { + for i, c := range chars { + for _, f := range c.forms { + isNo := (f.quickCheck[MDecomposed] == QCNo) + if (len(f.decomp) > 0) != isNo && !isHangul(rune(i)) { + log.Fatalf("%U: NF*D QC must be No if rune decomposes", i) + } + + isMaybe := f.quickCheck[MComposed] == QCMaybe + if f.combinesBackward != isMaybe { + log.Fatalf("%U: NF*C QC must be Maybe if combinesBackward", i) + } + if len(f.decomp) > 0 && f.combinesForward && isMaybe { + log.Fatalf("%U: NF*C QC must be Yes or No if combinesForward and decomposes", i) + } + + if len(f.expandedDecomp) != 0 { + continue + } + if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b { + // We accept these runes to be treated differently (it only affects + // segment breaking in iteration, most likely on improper use), but + // reconsider if more characters are added. + // U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L; 3099;;;;N;;;;; + // U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L; 309A;;;;N;;;;; + // U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L; 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;; + // U+318E HANGUL LETTER ARAEAE;Lo;0;L; 11A1;;;;N;HANGUL LETTER ALAE AE;;;; + // U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L; 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;; + // U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L; 3163;;;;N;;;;; + if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) { + log.Fatalf("%U: nLead was %v; want %v", i, a, b) + } + } + } + nfc := c.forms[FCanonical] + nfkc := c.forms[FCompatibility] + if nfc.combinesBackward != nfkc.combinesBackward { + log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint) + } + } +} + +// Use values in DerivedNormalizationProps.txt to compare against the +// values we computed. +// DerivedNormalizationProps.txt has form: +// 00C0..00C5 ; NFD_QC; N # ... +// 0374 ; NFD_QC; N # ... +// See http://unicode.org/reports/tr44/ for full explanation +func testDerived() { + f := gen.OpenUCDFile("DerivedNormalizationProps.txt") + defer f.Close() + p := ucd.New(f) + for p.Next() { + r := p.Rune(0) + c := &chars[r] + + var ftype, mode int + qt := p.String(1) + switch qt { + case "NFC_QC": + ftype, mode = FCanonical, MComposed + case "NFD_QC": + ftype, mode = FCanonical, MDecomposed + case "NFKC_QC": + ftype, mode = FCompatibility, MComposed + case "NFKD_QC": + ftype, mode = FCompatibility, MDecomposed + default: + continue + } + var qr QCResult + switch p.String(2) { + case "Y": + qr = QCYes + case "N": + qr = QCNo + case "M": + qr = QCMaybe + default: + log.Fatalf(`Unexpected quick check value "%s"`, p.String(2)) + } + if got := c.forms[ftype].quickCheck[mode]; got != qr { + log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr) + } + c.forms[ftype].verified[mode] = true + } + if err := p.Err(); err != nil { + log.Fatal(err) + } + // Any unspecified value must be QCYes. Verify this. + for i, c := range chars { + for j, fd := range c.forms { + for k, qr := range fd.quickCheck { + if !fd.verified[k] && qr != QCYes { + m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n" + log.Printf(m, i, j, k, qr, c.name) + } + } + } + } +} + +var testHeader = `const ( + Yes = iota + No + Maybe +) + +type formData struct { + qc uint8 + combinesForward bool + decomposition string +} + +type runeData struct { + r rune + ccc uint8 + nLead uint8 + nTrail uint8 + f [2]formData // 0: canonical; 1: compatibility +} + +func f(qc uint8, cf bool, dec string) [2]formData { + return [2]formData{{qc, cf, dec}, {qc, cf, dec}} +} + +func g(qc, qck uint8, cf, cfk bool, d, dk string) [2]formData { + return [2]formData{{qc, cf, d}, {qck, cfk, dk}} +} + +var testData = []runeData{ +` + +func printTestdata() { + type lastInfo struct { + ccc uint8 + nLead uint8 + nTrail uint8 + f string + } + + last := lastInfo{} + w := &bytes.Buffer{} + fmt.Fprintf(w, testHeader) + for r, c := range chars { + f := c.forms[FCanonical] + qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) + f = c.forms[FCompatibility] + qck, cfk, dk := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) + s := "" + if d == dk && qc == qck && cf == cfk { + s = fmt.Sprintf("f(%s, %v, %q)", qc, cf, d) + } else { + s = fmt.Sprintf("g(%s, %s, %v, %v, %q, %q)", qc, qck, cf, cfk, d, dk) + } + current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s} + if last != current { + fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s) + last = current + } + } + fmt.Fprintln(w, "}") + gen.WriteGoFile("data_test.go", "norm", w.Bytes()) +} diff --git a/vendor/golang.org/x/text/unicode/norm/normalize.go b/vendor/golang.org/x/text/unicode/norm/normalize.go new file mode 100644 index 00000000000..bba8ce9f95d --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/normalize.go @@ -0,0 +1,608 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:generate go run maketables.go triegen.go +//go:generate go run maketables.go triegen.go -test + +// Package norm contains types and functions for normalizing Unicode strings. +package norm // import "golang.org/x/text/unicode/norm" + +import ( + "unicode/utf8" + + "golang.org/x/text/transform" +) + +// A Form denotes a canonical representation of Unicode code points. +// The Unicode-defined normalization and equivalence forms are: +// +// NFC Unicode Normalization Form C +// NFD Unicode Normalization Form D +// NFKC Unicode Normalization Form KC +// NFKD Unicode Normalization Form KD +// +// For a Form f, this documentation uses the notation f(x) to mean +// the bytes or string x converted to the given form. +// A position n in x is called a boundary if conversion to the form can +// proceed independently on both sides: +// f(x) == append(f(x[0:n]), f(x[n:])...) +// +// References: http://unicode.org/reports/tr15/ and +// http://unicode.org/notes/tn5/. +type Form int + +const ( + NFC Form = iota + NFD + NFKC + NFKD +) + +// Bytes returns f(b). May return b if f(b) = b. +func (f Form) Bytes(b []byte) []byte { + src := inputBytes(b) + ft := formTable[f] + n, ok := ft.quickSpan(src, 0, len(b), true) + if ok { + return b + } + out := make([]byte, n, len(b)) + copy(out, b[0:n]) + rb := reorderBuffer{f: *ft, src: src, nsrc: len(b), out: out, flushF: appendFlush} + return doAppendInner(&rb, n) +} + +// String returns f(s). +func (f Form) String(s string) string { + src := inputString(s) + ft := formTable[f] + n, ok := ft.quickSpan(src, 0, len(s), true) + if ok { + return s + } + out := make([]byte, n, len(s)) + copy(out, s[0:n]) + rb := reorderBuffer{f: *ft, src: src, nsrc: len(s), out: out, flushF: appendFlush} + return string(doAppendInner(&rb, n)) +} + +// IsNormal returns true if b == f(b). +func (f Form) IsNormal(b []byte) bool { + src := inputBytes(b) + ft := formTable[f] + bp, ok := ft.quickSpan(src, 0, len(b), true) + if ok { + return true + } + rb := reorderBuffer{f: *ft, src: src, nsrc: len(b)} + rb.setFlusher(nil, cmpNormalBytes) + for bp < len(b) { + rb.out = b[bp:] + if bp = decomposeSegment(&rb, bp, true); bp < 0 { + return false + } + bp, _ = rb.f.quickSpan(rb.src, bp, len(b), true) + } + return true +} + +func cmpNormalBytes(rb *reorderBuffer) bool { + b := rb.out + for i := 0; i < rb.nrune; i++ { + info := rb.rune[i] + if int(info.size) > len(b) { + return false + } + p := info.pos + pe := p + info.size + for ; p < pe; p++ { + if b[0] != rb.byte[p] { + return false + } + b = b[1:] + } + } + return true +} + +// IsNormalString returns true if s == f(s). +func (f Form) IsNormalString(s string) bool { + src := inputString(s) + ft := formTable[f] + bp, ok := ft.quickSpan(src, 0, len(s), true) + if ok { + return true + } + rb := reorderBuffer{f: *ft, src: src, nsrc: len(s)} + rb.setFlusher(nil, func(rb *reorderBuffer) bool { + for i := 0; i < rb.nrune; i++ { + info := rb.rune[i] + if bp+int(info.size) > len(s) { + return false + } + p := info.pos + pe := p + info.size + for ; p < pe; p++ { + if s[bp] != rb.byte[p] { + return false + } + bp++ + } + } + return true + }) + for bp < len(s) { + if bp = decomposeSegment(&rb, bp, true); bp < 0 { + return false + } + bp, _ = rb.f.quickSpan(rb.src, bp, len(s), true) + } + return true +} + +// patchTail fixes a case where a rune may be incorrectly normalized +// if it is followed by illegal continuation bytes. It returns the +// patched buffer and whether the decomposition is still in progress. +func patchTail(rb *reorderBuffer) bool { + info, p := lastRuneStart(&rb.f, rb.out) + if p == -1 || info.size == 0 { + return true + } + end := p + int(info.size) + extra := len(rb.out) - end + if extra > 0 { + // Potentially allocating memory. However, this only + // happens with ill-formed UTF-8. + x := make([]byte, 0) + x = append(x, rb.out[len(rb.out)-extra:]...) + rb.out = rb.out[:end] + decomposeToLastBoundary(rb) + rb.doFlush() + rb.out = append(rb.out, x...) + return false + } + buf := rb.out[p:] + rb.out = rb.out[:p] + decomposeToLastBoundary(rb) + if s := rb.ss.next(info); s == ssStarter { + rb.doFlush() + rb.ss.first(info) + } else if s == ssOverflow { + rb.doFlush() + rb.insertCGJ() + rb.ss = 0 + } + rb.insertUnsafe(inputBytes(buf), 0, info) + return true +} + +func appendQuick(rb *reorderBuffer, i int) int { + if rb.nsrc == i { + return i + } + end, _ := rb.f.quickSpan(rb.src, i, rb.nsrc, true) + rb.out = rb.src.appendSlice(rb.out, i, end) + return end +} + +// Append returns f(append(out, b...)). +// The buffer out must be nil, empty, or equal to f(out). +func (f Form) Append(out []byte, src ...byte) []byte { + return f.doAppend(out, inputBytes(src), len(src)) +} + +func (f Form) doAppend(out []byte, src input, n int) []byte { + if n == 0 { + return out + } + ft := formTable[f] + // Attempt to do a quickSpan first so we can avoid initializing the reorderBuffer. + if len(out) == 0 { + p, _ := ft.quickSpan(src, 0, n, true) + out = src.appendSlice(out, 0, p) + if p == n { + return out + } + rb := reorderBuffer{f: *ft, src: src, nsrc: n, out: out, flushF: appendFlush} + return doAppendInner(&rb, p) + } + rb := reorderBuffer{f: *ft, src: src, nsrc: n} + return doAppend(&rb, out, 0) +} + +func doAppend(rb *reorderBuffer, out []byte, p int) []byte { + rb.setFlusher(out, appendFlush) + src, n := rb.src, rb.nsrc + doMerge := len(out) > 0 + if q := src.skipContinuationBytes(p); q > p { + // Move leading non-starters to destination. + rb.out = src.appendSlice(rb.out, p, q) + p = q + doMerge = patchTail(rb) + } + fd := &rb.f + if doMerge { + var info Properties + if p < n { + info = fd.info(src, p) + if !info.BoundaryBefore() || info.nLeadingNonStarters() > 0 { + if p == 0 { + decomposeToLastBoundary(rb) + } + p = decomposeSegment(rb, p, true) + } + } + if info.size == 0 { + rb.doFlush() + // Append incomplete UTF-8 encoding. + return src.appendSlice(rb.out, p, n) + } + if rb.nrune > 0 { + return doAppendInner(rb, p) + } + } + p = appendQuick(rb, p) + return doAppendInner(rb, p) +} + +func doAppendInner(rb *reorderBuffer, p int) []byte { + for n := rb.nsrc; p < n; { + p = decomposeSegment(rb, p, true) + p = appendQuick(rb, p) + } + return rb.out +} + +// AppendString returns f(append(out, []byte(s))). +// The buffer out must be nil, empty, or equal to f(out). +func (f Form) AppendString(out []byte, src string) []byte { + return f.doAppend(out, inputString(src), len(src)) +} + +// QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]). +// It is not guaranteed to return the largest such n. +func (f Form) QuickSpan(b []byte) int { + n, _ := formTable[f].quickSpan(inputBytes(b), 0, len(b), true) + return n +} + +// Span implements transform.SpanningTransformer. It returns a boundary n such +// that b[0:n] == f(b[0:n]). It is not guaranteed to return the largest such n. +func (f Form) Span(b []byte, atEOF bool) (n int, err error) { + n, ok := formTable[f].quickSpan(inputBytes(b), 0, len(b), atEOF) + if n < len(b) { + if !ok { + err = transform.ErrEndOfSpan + } else { + err = transform.ErrShortSrc + } + } + return n, err +} + +// SpanString returns a boundary n such that s[0:n] == f(s[0:n]). +// It is not guaranteed to return the largest such n. +func (f Form) SpanString(s string, atEOF bool) (n int, err error) { + n, ok := formTable[f].quickSpan(inputString(s), 0, len(s), atEOF) + if n < len(s) { + if !ok { + err = transform.ErrEndOfSpan + } else { + err = transform.ErrShortSrc + } + } + return n, err +} + +// quickSpan returns a boundary n such that src[0:n] == f(src[0:n]) and +// whether any non-normalized parts were found. If atEOF is false, n will +// not point past the last segment if this segment might be become +// non-normalized by appending other runes. +func (f *formInfo) quickSpan(src input, i, end int, atEOF bool) (n int, ok bool) { + var lastCC uint8 + ss := streamSafe(0) + lastSegStart := i + for n = end; i < n; { + if j := src.skipASCII(i, n); i != j { + i = j + lastSegStart = i - 1 + lastCC = 0 + ss = 0 + continue + } + info := f.info(src, i) + if info.size == 0 { + if atEOF { + // include incomplete runes + return n, true + } + return lastSegStart, true + } + // This block needs to be before the next, because it is possible to + // have an overflow for runes that are starters (e.g. with U+FF9E). + switch ss.next(info) { + case ssStarter: + ss.first(info) + lastSegStart = i + case ssOverflow: + return lastSegStart, false + case ssSuccess: + if lastCC > info.ccc { + return lastSegStart, false + } + } + if f.composing { + if !info.isYesC() { + break + } + } else { + if !info.isYesD() { + break + } + } + lastCC = info.ccc + i += int(info.size) + } + if i == n { + if !atEOF { + n = lastSegStart + } + return n, true + } + return lastSegStart, false +} + +// QuickSpanString returns a boundary n such that s[0:n] == f(s[0:n]). +// It is not guaranteed to return the largest such n. +func (f Form) QuickSpanString(s string) int { + n, _ := formTable[f].quickSpan(inputString(s), 0, len(s), true) + return n +} + +// FirstBoundary returns the position i of the first boundary in b +// or -1 if b contains no boundary. +func (f Form) FirstBoundary(b []byte) int { + return f.firstBoundary(inputBytes(b), len(b)) +} + +func (f Form) firstBoundary(src input, nsrc int) int { + i := src.skipContinuationBytes(0) + if i >= nsrc { + return -1 + } + fd := formTable[f] + ss := streamSafe(0) + // We should call ss.first here, but we can't as the first rune is + // skipped already. This means FirstBoundary can't really determine + // CGJ insertion points correctly. Luckily it doesn't have to. + for { + info := fd.info(src, i) + if info.size == 0 { + return -1 + } + if s := ss.next(info); s != ssSuccess { + return i + } + i += int(info.size) + if i >= nsrc { + if !info.BoundaryAfter() && !ss.isMax() { + return -1 + } + return nsrc + } + } +} + +// FirstBoundaryInString returns the position i of the first boundary in s +// or -1 if s contains no boundary. +func (f Form) FirstBoundaryInString(s string) int { + return f.firstBoundary(inputString(s), len(s)) +} + +// NextBoundary reports the index of the boundary between the first and next +// segment in b or -1 if atEOF is false and there are not enough bytes to +// determine this boundary. +func (f Form) NextBoundary(b []byte, atEOF bool) int { + return f.nextBoundary(inputBytes(b), len(b), atEOF) +} + +// NextBoundaryInString reports the index of the boundary between the first and +// next segment in b or -1 if atEOF is false and there are not enough bytes to +// determine this boundary. +func (f Form) NextBoundaryInString(s string, atEOF bool) int { + return f.nextBoundary(inputString(s), len(s), atEOF) +} + +func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int { + if nsrc == 0 { + if atEOF { + return 0 + } + return -1 + } + fd := formTable[f] + info := fd.info(src, 0) + if info.size == 0 { + if atEOF { + return 1 + } + return -1 + } + ss := streamSafe(0) + ss.first(info) + + for i := int(info.size); i < nsrc; i += int(info.size) { + info = fd.info(src, i) + if info.size == 0 { + if atEOF { + return i + } + return -1 + } + if s := ss.next(info); s != ssSuccess { + return i + } + } + if !atEOF && !info.BoundaryAfter() && !ss.isMax() { + return -1 + } + return nsrc +} + +// LastBoundary returns the position i of the last boundary in b +// or -1 if b contains no boundary. +func (f Form) LastBoundary(b []byte) int { + return lastBoundary(formTable[f], b) +} + +func lastBoundary(fd *formInfo, b []byte) int { + i := len(b) + info, p := lastRuneStart(fd, b) + if p == -1 { + return -1 + } + if info.size == 0 { // ends with incomplete rune + if p == 0 { // starts with incomplete rune + return -1 + } + i = p + info, p = lastRuneStart(fd, b[:i]) + if p == -1 { // incomplete UTF-8 encoding or non-starter bytes without a starter + return i + } + } + if p+int(info.size) != i { // trailing non-starter bytes: illegal UTF-8 + return i + } + if info.BoundaryAfter() { + return i + } + ss := streamSafe(0) + v := ss.backwards(info) + for i = p; i >= 0 && v != ssStarter; i = p { + info, p = lastRuneStart(fd, b[:i]) + if v = ss.backwards(info); v == ssOverflow { + break + } + if p+int(info.size) != i { + if p == -1 { // no boundary found + return -1 + } + return i // boundary after an illegal UTF-8 encoding + } + } + return i +} + +// decomposeSegment scans the first segment in src into rb. It inserts 0x034f +// (Grapheme Joiner) when it encounters a sequence of more than 30 non-starters +// and returns the number of bytes consumed from src or iShortDst or iShortSrc. +func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int { + // Force one character to be consumed. + info := rb.f.info(rb.src, sp) + if info.size == 0 { + return 0 + } + if rb.nrune > 0 { + if s := rb.ss.next(info); s == ssStarter { + goto end + } else if s == ssOverflow { + rb.insertCGJ() + goto end + } + } else { + rb.ss.first(info) + } + if err := rb.insertFlush(rb.src, sp, info); err != iSuccess { + return int(err) + } + for { + sp += int(info.size) + if sp >= rb.nsrc { + if !atEOF && !info.BoundaryAfter() { + return int(iShortSrc) + } + break + } + info = rb.f.info(rb.src, sp) + if info.size == 0 { + if !atEOF { + return int(iShortSrc) + } + break + } + if s := rb.ss.next(info); s == ssStarter { + break + } else if s == ssOverflow { + rb.insertCGJ() + break + } + if err := rb.insertFlush(rb.src, sp, info); err != iSuccess { + return int(err) + } + } +end: + if !rb.doFlush() { + return int(iShortDst) + } + return sp +} + +// lastRuneStart returns the runeInfo and position of the last +// rune in buf or the zero runeInfo and -1 if no rune was found. +func lastRuneStart(fd *formInfo, buf []byte) (Properties, int) { + p := len(buf) - 1 + for ; p >= 0 && !utf8.RuneStart(buf[p]); p-- { + } + if p < 0 { + return Properties{}, -1 + } + return fd.info(inputBytes(buf), p), p +} + +// decomposeToLastBoundary finds an open segment at the end of the buffer +// and scans it into rb. Returns the buffer minus the last segment. +func decomposeToLastBoundary(rb *reorderBuffer) { + fd := &rb.f + info, i := lastRuneStart(fd, rb.out) + if int(info.size) != len(rb.out)-i { + // illegal trailing continuation bytes + return + } + if info.BoundaryAfter() { + return + } + var add [maxNonStarters + 1]Properties // stores runeInfo in reverse order + padd := 0 + ss := streamSafe(0) + p := len(rb.out) + for { + add[padd] = info + v := ss.backwards(info) + if v == ssOverflow { + // Note that if we have an overflow, it the string we are appending to + // is not correctly normalized. In this case the behavior is undefined. + break + } + padd++ + p -= int(info.size) + if v == ssStarter || p < 0 { + break + } + info, i = lastRuneStart(fd, rb.out[:p]) + if int(info.size) != p-i { + break + } + } + rb.ss = ss + // Copy bytes for insertion as we may need to overwrite rb.out. + var buf [maxBufferSize * utf8.UTFMax]byte + cp := buf[:copy(buf[:], rb.out[p:])] + rb.out = rb.out[:p] + for padd--; padd >= 0; padd-- { + info = add[padd] + rb.insertUnsafe(inputBytes(cp), 0, info) + cp = cp[info.size:] + } +} diff --git a/vendor/golang.org/x/text/unicode/norm/readwriter.go b/vendor/golang.org/x/text/unicode/norm/readwriter.go new file mode 100644 index 00000000000..d926ee903e5 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/readwriter.go @@ -0,0 +1,125 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import "io" + +type normWriter struct { + rb reorderBuffer + w io.Writer + buf []byte +} + +// Write implements the standard write interface. If the last characters are +// not at a normalization boundary, the bytes will be buffered for the next +// write. The remaining bytes will be written on close. +func (w *normWriter) Write(data []byte) (n int, err error) { + // Process data in pieces to keep w.buf size bounded. + const chunk = 4000 + + for len(data) > 0 { + // Normalize into w.buf. + m := len(data) + if m > chunk { + m = chunk + } + w.rb.src = inputBytes(data[:m]) + w.rb.nsrc = m + w.buf = doAppend(&w.rb, w.buf, 0) + data = data[m:] + n += m + + // Write out complete prefix, save remainder. + // Note that lastBoundary looks back at most 31 runes. + i := lastBoundary(&w.rb.f, w.buf) + if i == -1 { + i = 0 + } + if i > 0 { + if _, err = w.w.Write(w.buf[:i]); err != nil { + break + } + bn := copy(w.buf, w.buf[i:]) + w.buf = w.buf[:bn] + } + } + return n, err +} + +// Close forces data that remains in the buffer to be written. +func (w *normWriter) Close() error { + if len(w.buf) > 0 { + _, err := w.w.Write(w.buf) + if err != nil { + return err + } + } + return nil +} + +// Writer returns a new writer that implements Write(b) +// by writing f(b) to w. The returned writer may use an +// an internal buffer to maintain state across Write calls. +// Calling its Close method writes any buffered data to w. +func (f Form) Writer(w io.Writer) io.WriteCloser { + wr := &normWriter{rb: reorderBuffer{}, w: w} + wr.rb.init(f, nil) + return wr +} + +type normReader struct { + rb reorderBuffer + r io.Reader + inbuf []byte + outbuf []byte + bufStart int + lastBoundary int + err error +} + +// Read implements the standard read interface. +func (r *normReader) Read(p []byte) (int, error) { + for { + if r.lastBoundary-r.bufStart > 0 { + n := copy(p, r.outbuf[r.bufStart:r.lastBoundary]) + r.bufStart += n + if r.lastBoundary-r.bufStart > 0 { + return n, nil + } + return n, r.err + } + if r.err != nil { + return 0, r.err + } + outn := copy(r.outbuf, r.outbuf[r.lastBoundary:]) + r.outbuf = r.outbuf[0:outn] + r.bufStart = 0 + + n, err := r.r.Read(r.inbuf) + r.rb.src = inputBytes(r.inbuf[0:n]) + r.rb.nsrc, r.err = n, err + if n > 0 { + r.outbuf = doAppend(&r.rb, r.outbuf, 0) + } + if err == io.EOF { + r.lastBoundary = len(r.outbuf) + } else { + r.lastBoundary = lastBoundary(&r.rb.f, r.outbuf) + if r.lastBoundary == -1 { + r.lastBoundary = 0 + } + } + } +} + +// Reader returns a new reader that implements Read +// by reading data from r and returning f(data). +func (f Form) Reader(r io.Reader) io.Reader { + const chunk = 4000 + buf := make([]byte, chunk) + rr := &normReader{rb: reorderBuffer{}, r: r, inbuf: buf} + rr.rb.init(f, buf) + return rr +} diff --git a/vendor/golang.org/x/text/unicode/norm/tables.go b/vendor/golang.org/x/text/unicode/norm/tables.go new file mode 100644 index 00000000000..a56697b571b --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/tables.go @@ -0,0 +1,7627 @@ +// This file was generated by go generate; DO NOT EDIT + +package norm + +const ( + // Version is the Unicode edition from which the tables are derived. + Version = "9.0.0" + + // MaxTransformChunkSize indicates the maximum number of bytes that Transform + // may need to write atomically for any Form. Making a destination buffer at + // least this size ensures that Transform can always make progress and that + // the user does not need to grow the buffer on an ErrShortDst. + MaxTransformChunkSize = 35 + maxNonStarters*4 +) + +var ccc = [55]uint8{ + 0, 1, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, + 84, 91, 103, 107, 118, 122, 129, 130, + 132, 202, 214, 216, 218, 220, 222, 224, + 226, 228, 230, 232, 233, 234, 240, +} + +const ( + firstMulti = 0x186D + firstCCC = 0x2C9E + endMulti = 0x2F60 + firstLeadingCCC = 0x4A44 + firstCCCZeroExcept = 0x4A5A + firstStarterWithNLead = 0x4A81 + lastDecomp = 0x4A83 + maxDecomp = 0x8000 +) + +// decomps: 19075 bytes +var decomps = [...]byte{ + // Bytes 0 - 3f + 0x00, 0x41, 0x20, 0x41, 0x21, 0x41, 0x22, 0x41, + 0x23, 0x41, 0x24, 0x41, 0x25, 0x41, 0x26, 0x41, + 0x27, 0x41, 0x28, 0x41, 0x29, 0x41, 0x2A, 0x41, + 0x2B, 0x41, 0x2C, 0x41, 0x2D, 0x41, 0x2E, 0x41, + 0x2F, 0x41, 0x30, 0x41, 0x31, 0x41, 0x32, 0x41, + 0x33, 0x41, 0x34, 0x41, 0x35, 0x41, 0x36, 0x41, + 0x37, 0x41, 0x38, 0x41, 0x39, 0x41, 0x3A, 0x41, + 0x3B, 0x41, 0x3C, 0x41, 0x3D, 0x41, 0x3E, 0x41, + // Bytes 40 - 7f + 0x3F, 0x41, 0x40, 0x41, 0x41, 0x41, 0x42, 0x41, + 0x43, 0x41, 0x44, 0x41, 0x45, 0x41, 0x46, 0x41, + 0x47, 0x41, 0x48, 0x41, 0x49, 0x41, 0x4A, 0x41, + 0x4B, 0x41, 0x4C, 0x41, 0x4D, 0x41, 0x4E, 0x41, + 0x4F, 0x41, 0x50, 0x41, 0x51, 0x41, 0x52, 0x41, + 0x53, 0x41, 0x54, 0x41, 0x55, 0x41, 0x56, 0x41, + 0x57, 0x41, 0x58, 0x41, 0x59, 0x41, 0x5A, 0x41, + 0x5B, 0x41, 0x5C, 0x41, 0x5D, 0x41, 0x5E, 0x41, + // Bytes 80 - bf + 0x5F, 0x41, 0x60, 0x41, 0x61, 0x41, 0x62, 0x41, + 0x63, 0x41, 0x64, 0x41, 0x65, 0x41, 0x66, 0x41, + 0x67, 0x41, 0x68, 0x41, 0x69, 0x41, 0x6A, 0x41, + 0x6B, 0x41, 0x6C, 0x41, 0x6D, 0x41, 0x6E, 0x41, + 0x6F, 0x41, 0x70, 0x41, 0x71, 0x41, 0x72, 0x41, + 0x73, 0x41, 0x74, 0x41, 0x75, 0x41, 0x76, 0x41, + 0x77, 0x41, 0x78, 0x41, 0x79, 0x41, 0x7A, 0x41, + 0x7B, 0x41, 0x7C, 0x41, 0x7D, 0x41, 0x7E, 0x42, + // Bytes c0 - ff + 0xC2, 0xA2, 0x42, 0xC2, 0xA3, 0x42, 0xC2, 0xA5, + 0x42, 0xC2, 0xA6, 0x42, 0xC2, 0xAC, 0x42, 0xC2, + 0xB7, 0x42, 0xC3, 0x86, 0x42, 0xC3, 0xB0, 0x42, + 0xC4, 0xA6, 0x42, 0xC4, 0xA7, 0x42, 0xC4, 0xB1, + 0x42, 0xC5, 0x8B, 0x42, 0xC5, 0x93, 0x42, 0xC6, + 0x8E, 0x42, 0xC6, 0x90, 0x42, 0xC6, 0xAB, 0x42, + 0xC8, 0xA2, 0x42, 0xC8, 0xB7, 0x42, 0xC9, 0x90, + 0x42, 0xC9, 0x91, 0x42, 0xC9, 0x92, 0x42, 0xC9, + // Bytes 100 - 13f + 0x94, 0x42, 0xC9, 0x95, 0x42, 0xC9, 0x99, 0x42, + 0xC9, 0x9B, 0x42, 0xC9, 0x9C, 0x42, 0xC9, 0x9F, + 0x42, 0xC9, 0xA1, 0x42, 0xC9, 0xA3, 0x42, 0xC9, + 0xA5, 0x42, 0xC9, 0xA6, 0x42, 0xC9, 0xA8, 0x42, + 0xC9, 0xA9, 0x42, 0xC9, 0xAA, 0x42, 0xC9, 0xAB, + 0x42, 0xC9, 0xAD, 0x42, 0xC9, 0xAF, 0x42, 0xC9, + 0xB0, 0x42, 0xC9, 0xB1, 0x42, 0xC9, 0xB2, 0x42, + 0xC9, 0xB3, 0x42, 0xC9, 0xB4, 0x42, 0xC9, 0xB5, + // Bytes 140 - 17f + 0x42, 0xC9, 0xB8, 0x42, 0xC9, 0xB9, 0x42, 0xC9, + 0xBB, 0x42, 0xCA, 0x81, 0x42, 0xCA, 0x82, 0x42, + 0xCA, 0x83, 0x42, 0xCA, 0x89, 0x42, 0xCA, 0x8A, + 0x42, 0xCA, 0x8B, 0x42, 0xCA, 0x8C, 0x42, 0xCA, + 0x90, 0x42, 0xCA, 0x91, 0x42, 0xCA, 0x92, 0x42, + 0xCA, 0x95, 0x42, 0xCA, 0x9D, 0x42, 0xCA, 0x9F, + 0x42, 0xCA, 0xB9, 0x42, 0xCE, 0x91, 0x42, 0xCE, + 0x92, 0x42, 0xCE, 0x93, 0x42, 0xCE, 0x94, 0x42, + // Bytes 180 - 1bf + 0xCE, 0x95, 0x42, 0xCE, 0x96, 0x42, 0xCE, 0x97, + 0x42, 0xCE, 0x98, 0x42, 0xCE, 0x99, 0x42, 0xCE, + 0x9A, 0x42, 0xCE, 0x9B, 0x42, 0xCE, 0x9C, 0x42, + 0xCE, 0x9D, 0x42, 0xCE, 0x9E, 0x42, 0xCE, 0x9F, + 0x42, 0xCE, 0xA0, 0x42, 0xCE, 0xA1, 0x42, 0xCE, + 0xA3, 0x42, 0xCE, 0xA4, 0x42, 0xCE, 0xA5, 0x42, + 0xCE, 0xA6, 0x42, 0xCE, 0xA7, 0x42, 0xCE, 0xA8, + 0x42, 0xCE, 0xA9, 0x42, 0xCE, 0xB1, 0x42, 0xCE, + // Bytes 1c0 - 1ff + 0xB2, 0x42, 0xCE, 0xB3, 0x42, 0xCE, 0xB4, 0x42, + 0xCE, 0xB5, 0x42, 0xCE, 0xB6, 0x42, 0xCE, 0xB7, + 0x42, 0xCE, 0xB8, 0x42, 0xCE, 0xB9, 0x42, 0xCE, + 0xBA, 0x42, 0xCE, 0xBB, 0x42, 0xCE, 0xBC, 0x42, + 0xCE, 0xBD, 0x42, 0xCE, 0xBE, 0x42, 0xCE, 0xBF, + 0x42, 0xCF, 0x80, 0x42, 0xCF, 0x81, 0x42, 0xCF, + 0x82, 0x42, 0xCF, 0x83, 0x42, 0xCF, 0x84, 0x42, + 0xCF, 0x85, 0x42, 0xCF, 0x86, 0x42, 0xCF, 0x87, + // Bytes 200 - 23f + 0x42, 0xCF, 0x88, 0x42, 0xCF, 0x89, 0x42, 0xCF, + 0x9C, 0x42, 0xCF, 0x9D, 0x42, 0xD0, 0xBD, 0x42, + 0xD1, 0x8A, 0x42, 0xD1, 0x8C, 0x42, 0xD7, 0x90, + 0x42, 0xD7, 0x91, 0x42, 0xD7, 0x92, 0x42, 0xD7, + 0x93, 0x42, 0xD7, 0x94, 0x42, 0xD7, 0x9B, 0x42, + 0xD7, 0x9C, 0x42, 0xD7, 0x9D, 0x42, 0xD7, 0xA2, + 0x42, 0xD7, 0xA8, 0x42, 0xD7, 0xAA, 0x42, 0xD8, + 0xA1, 0x42, 0xD8, 0xA7, 0x42, 0xD8, 0xA8, 0x42, + // Bytes 240 - 27f + 0xD8, 0xA9, 0x42, 0xD8, 0xAA, 0x42, 0xD8, 0xAB, + 0x42, 0xD8, 0xAC, 0x42, 0xD8, 0xAD, 0x42, 0xD8, + 0xAE, 0x42, 0xD8, 0xAF, 0x42, 0xD8, 0xB0, 0x42, + 0xD8, 0xB1, 0x42, 0xD8, 0xB2, 0x42, 0xD8, 0xB3, + 0x42, 0xD8, 0xB4, 0x42, 0xD8, 0xB5, 0x42, 0xD8, + 0xB6, 0x42, 0xD8, 0xB7, 0x42, 0xD8, 0xB8, 0x42, + 0xD8, 0xB9, 0x42, 0xD8, 0xBA, 0x42, 0xD9, 0x81, + 0x42, 0xD9, 0x82, 0x42, 0xD9, 0x83, 0x42, 0xD9, + // Bytes 280 - 2bf + 0x84, 0x42, 0xD9, 0x85, 0x42, 0xD9, 0x86, 0x42, + 0xD9, 0x87, 0x42, 0xD9, 0x88, 0x42, 0xD9, 0x89, + 0x42, 0xD9, 0x8A, 0x42, 0xD9, 0xAE, 0x42, 0xD9, + 0xAF, 0x42, 0xD9, 0xB1, 0x42, 0xD9, 0xB9, 0x42, + 0xD9, 0xBA, 0x42, 0xD9, 0xBB, 0x42, 0xD9, 0xBE, + 0x42, 0xD9, 0xBF, 0x42, 0xDA, 0x80, 0x42, 0xDA, + 0x83, 0x42, 0xDA, 0x84, 0x42, 0xDA, 0x86, 0x42, + 0xDA, 0x87, 0x42, 0xDA, 0x88, 0x42, 0xDA, 0x8C, + // Bytes 2c0 - 2ff + 0x42, 0xDA, 0x8D, 0x42, 0xDA, 0x8E, 0x42, 0xDA, + 0x91, 0x42, 0xDA, 0x98, 0x42, 0xDA, 0xA1, 0x42, + 0xDA, 0xA4, 0x42, 0xDA, 0xA6, 0x42, 0xDA, 0xA9, + 0x42, 0xDA, 0xAD, 0x42, 0xDA, 0xAF, 0x42, 0xDA, + 0xB1, 0x42, 0xDA, 0xB3, 0x42, 0xDA, 0xBA, 0x42, + 0xDA, 0xBB, 0x42, 0xDA, 0xBE, 0x42, 0xDB, 0x81, + 0x42, 0xDB, 0x85, 0x42, 0xDB, 0x86, 0x42, 0xDB, + 0x87, 0x42, 0xDB, 0x88, 0x42, 0xDB, 0x89, 0x42, + // Bytes 300 - 33f + 0xDB, 0x8B, 0x42, 0xDB, 0x8C, 0x42, 0xDB, 0x90, + 0x42, 0xDB, 0x92, 0x43, 0xE0, 0xBC, 0x8B, 0x43, + 0xE1, 0x83, 0x9C, 0x43, 0xE1, 0x84, 0x80, 0x43, + 0xE1, 0x84, 0x81, 0x43, 0xE1, 0x84, 0x82, 0x43, + 0xE1, 0x84, 0x83, 0x43, 0xE1, 0x84, 0x84, 0x43, + 0xE1, 0x84, 0x85, 0x43, 0xE1, 0x84, 0x86, 0x43, + 0xE1, 0x84, 0x87, 0x43, 0xE1, 0x84, 0x88, 0x43, + 0xE1, 0x84, 0x89, 0x43, 0xE1, 0x84, 0x8A, 0x43, + // Bytes 340 - 37f + 0xE1, 0x84, 0x8B, 0x43, 0xE1, 0x84, 0x8C, 0x43, + 0xE1, 0x84, 0x8D, 0x43, 0xE1, 0x84, 0x8E, 0x43, + 0xE1, 0x84, 0x8F, 0x43, 0xE1, 0x84, 0x90, 0x43, + 0xE1, 0x84, 0x91, 0x43, 0xE1, 0x84, 0x92, 0x43, + 0xE1, 0x84, 0x94, 0x43, 0xE1, 0x84, 0x95, 0x43, + 0xE1, 0x84, 0x9A, 0x43, 0xE1, 0x84, 0x9C, 0x43, + 0xE1, 0x84, 0x9D, 0x43, 0xE1, 0x84, 0x9E, 0x43, + 0xE1, 0x84, 0xA0, 0x43, 0xE1, 0x84, 0xA1, 0x43, + // Bytes 380 - 3bf + 0xE1, 0x84, 0xA2, 0x43, 0xE1, 0x84, 0xA3, 0x43, + 0xE1, 0x84, 0xA7, 0x43, 0xE1, 0x84, 0xA9, 0x43, + 0xE1, 0x84, 0xAB, 0x43, 0xE1, 0x84, 0xAC, 0x43, + 0xE1, 0x84, 0xAD, 0x43, 0xE1, 0x84, 0xAE, 0x43, + 0xE1, 0x84, 0xAF, 0x43, 0xE1, 0x84, 0xB2, 0x43, + 0xE1, 0x84, 0xB6, 0x43, 0xE1, 0x85, 0x80, 0x43, + 0xE1, 0x85, 0x87, 0x43, 0xE1, 0x85, 0x8C, 0x43, + 0xE1, 0x85, 0x97, 0x43, 0xE1, 0x85, 0x98, 0x43, + // Bytes 3c0 - 3ff + 0xE1, 0x85, 0x99, 0x43, 0xE1, 0x85, 0xA0, 0x43, + 0xE1, 0x86, 0x84, 0x43, 0xE1, 0x86, 0x85, 0x43, + 0xE1, 0x86, 0x88, 0x43, 0xE1, 0x86, 0x91, 0x43, + 0xE1, 0x86, 0x92, 0x43, 0xE1, 0x86, 0x94, 0x43, + 0xE1, 0x86, 0x9E, 0x43, 0xE1, 0x86, 0xA1, 0x43, + 0xE1, 0x87, 0x87, 0x43, 0xE1, 0x87, 0x88, 0x43, + 0xE1, 0x87, 0x8C, 0x43, 0xE1, 0x87, 0x8E, 0x43, + 0xE1, 0x87, 0x93, 0x43, 0xE1, 0x87, 0x97, 0x43, + // Bytes 400 - 43f + 0xE1, 0x87, 0x99, 0x43, 0xE1, 0x87, 0x9D, 0x43, + 0xE1, 0x87, 0x9F, 0x43, 0xE1, 0x87, 0xB1, 0x43, + 0xE1, 0x87, 0xB2, 0x43, 0xE1, 0xB4, 0x82, 0x43, + 0xE1, 0xB4, 0x96, 0x43, 0xE1, 0xB4, 0x97, 0x43, + 0xE1, 0xB4, 0x9C, 0x43, 0xE1, 0xB4, 0x9D, 0x43, + 0xE1, 0xB4, 0xA5, 0x43, 0xE1, 0xB5, 0xBB, 0x43, + 0xE1, 0xB6, 0x85, 0x43, 0xE2, 0x80, 0x82, 0x43, + 0xE2, 0x80, 0x83, 0x43, 0xE2, 0x80, 0x90, 0x43, + // Bytes 440 - 47f + 0xE2, 0x80, 0x93, 0x43, 0xE2, 0x80, 0x94, 0x43, + 0xE2, 0x82, 0xA9, 0x43, 0xE2, 0x86, 0x90, 0x43, + 0xE2, 0x86, 0x91, 0x43, 0xE2, 0x86, 0x92, 0x43, + 0xE2, 0x86, 0x93, 0x43, 0xE2, 0x88, 0x82, 0x43, + 0xE2, 0x88, 0x87, 0x43, 0xE2, 0x88, 0x91, 0x43, + 0xE2, 0x88, 0x92, 0x43, 0xE2, 0x94, 0x82, 0x43, + 0xE2, 0x96, 0xA0, 0x43, 0xE2, 0x97, 0x8B, 0x43, + 0xE2, 0xA6, 0x85, 0x43, 0xE2, 0xA6, 0x86, 0x43, + // Bytes 480 - 4bf + 0xE2, 0xB5, 0xA1, 0x43, 0xE3, 0x80, 0x81, 0x43, + 0xE3, 0x80, 0x82, 0x43, 0xE3, 0x80, 0x88, 0x43, + 0xE3, 0x80, 0x89, 0x43, 0xE3, 0x80, 0x8A, 0x43, + 0xE3, 0x80, 0x8B, 0x43, 0xE3, 0x80, 0x8C, 0x43, + 0xE3, 0x80, 0x8D, 0x43, 0xE3, 0x80, 0x8E, 0x43, + 0xE3, 0x80, 0x8F, 0x43, 0xE3, 0x80, 0x90, 0x43, + 0xE3, 0x80, 0x91, 0x43, 0xE3, 0x80, 0x92, 0x43, + 0xE3, 0x80, 0x94, 0x43, 0xE3, 0x80, 0x95, 0x43, + // Bytes 4c0 - 4ff + 0xE3, 0x80, 0x96, 0x43, 0xE3, 0x80, 0x97, 0x43, + 0xE3, 0x82, 0xA1, 0x43, 0xE3, 0x82, 0xA2, 0x43, + 0xE3, 0x82, 0xA3, 0x43, 0xE3, 0x82, 0xA4, 0x43, + 0xE3, 0x82, 0xA5, 0x43, 0xE3, 0x82, 0xA6, 0x43, + 0xE3, 0x82, 0xA7, 0x43, 0xE3, 0x82, 0xA8, 0x43, + 0xE3, 0x82, 0xA9, 0x43, 0xE3, 0x82, 0xAA, 0x43, + 0xE3, 0x82, 0xAB, 0x43, 0xE3, 0x82, 0xAD, 0x43, + 0xE3, 0x82, 0xAF, 0x43, 0xE3, 0x82, 0xB1, 0x43, + // Bytes 500 - 53f + 0xE3, 0x82, 0xB3, 0x43, 0xE3, 0x82, 0xB5, 0x43, + 0xE3, 0x82, 0xB7, 0x43, 0xE3, 0x82, 0xB9, 0x43, + 0xE3, 0x82, 0xBB, 0x43, 0xE3, 0x82, 0xBD, 0x43, + 0xE3, 0x82, 0xBF, 0x43, 0xE3, 0x83, 0x81, 0x43, + 0xE3, 0x83, 0x83, 0x43, 0xE3, 0x83, 0x84, 0x43, + 0xE3, 0x83, 0x86, 0x43, 0xE3, 0x83, 0x88, 0x43, + 0xE3, 0x83, 0x8A, 0x43, 0xE3, 0x83, 0x8B, 0x43, + 0xE3, 0x83, 0x8C, 0x43, 0xE3, 0x83, 0x8D, 0x43, + // Bytes 540 - 57f + 0xE3, 0x83, 0x8E, 0x43, 0xE3, 0x83, 0x8F, 0x43, + 0xE3, 0x83, 0x92, 0x43, 0xE3, 0x83, 0x95, 0x43, + 0xE3, 0x83, 0x98, 0x43, 0xE3, 0x83, 0x9B, 0x43, + 0xE3, 0x83, 0x9E, 0x43, 0xE3, 0x83, 0x9F, 0x43, + 0xE3, 0x83, 0xA0, 0x43, 0xE3, 0x83, 0xA1, 0x43, + 0xE3, 0x83, 0xA2, 0x43, 0xE3, 0x83, 0xA3, 0x43, + 0xE3, 0x83, 0xA4, 0x43, 0xE3, 0x83, 0xA5, 0x43, + 0xE3, 0x83, 0xA6, 0x43, 0xE3, 0x83, 0xA7, 0x43, + // Bytes 580 - 5bf + 0xE3, 0x83, 0xA8, 0x43, 0xE3, 0x83, 0xA9, 0x43, + 0xE3, 0x83, 0xAA, 0x43, 0xE3, 0x83, 0xAB, 0x43, + 0xE3, 0x83, 0xAC, 0x43, 0xE3, 0x83, 0xAD, 0x43, + 0xE3, 0x83, 0xAF, 0x43, 0xE3, 0x83, 0xB0, 0x43, + 0xE3, 0x83, 0xB1, 0x43, 0xE3, 0x83, 0xB2, 0x43, + 0xE3, 0x83, 0xB3, 0x43, 0xE3, 0x83, 0xBB, 0x43, + 0xE3, 0x83, 0xBC, 0x43, 0xE3, 0x92, 0x9E, 0x43, + 0xE3, 0x92, 0xB9, 0x43, 0xE3, 0x92, 0xBB, 0x43, + // Bytes 5c0 - 5ff + 0xE3, 0x93, 0x9F, 0x43, 0xE3, 0x94, 0x95, 0x43, + 0xE3, 0x9B, 0xAE, 0x43, 0xE3, 0x9B, 0xBC, 0x43, + 0xE3, 0x9E, 0x81, 0x43, 0xE3, 0xA0, 0xAF, 0x43, + 0xE3, 0xA1, 0xA2, 0x43, 0xE3, 0xA1, 0xBC, 0x43, + 0xE3, 0xA3, 0x87, 0x43, 0xE3, 0xA3, 0xA3, 0x43, + 0xE3, 0xA4, 0x9C, 0x43, 0xE3, 0xA4, 0xBA, 0x43, + 0xE3, 0xA8, 0xAE, 0x43, 0xE3, 0xA9, 0xAC, 0x43, + 0xE3, 0xAB, 0xA4, 0x43, 0xE3, 0xAC, 0x88, 0x43, + // Bytes 600 - 63f + 0xE3, 0xAC, 0x99, 0x43, 0xE3, 0xAD, 0x89, 0x43, + 0xE3, 0xAE, 0x9D, 0x43, 0xE3, 0xB0, 0x98, 0x43, + 0xE3, 0xB1, 0x8E, 0x43, 0xE3, 0xB4, 0xB3, 0x43, + 0xE3, 0xB6, 0x96, 0x43, 0xE3, 0xBA, 0xAC, 0x43, + 0xE3, 0xBA, 0xB8, 0x43, 0xE3, 0xBC, 0x9B, 0x43, + 0xE3, 0xBF, 0xBC, 0x43, 0xE4, 0x80, 0x88, 0x43, + 0xE4, 0x80, 0x98, 0x43, 0xE4, 0x80, 0xB9, 0x43, + 0xE4, 0x81, 0x86, 0x43, 0xE4, 0x82, 0x96, 0x43, + // Bytes 640 - 67f + 0xE4, 0x83, 0xA3, 0x43, 0xE4, 0x84, 0xAF, 0x43, + 0xE4, 0x88, 0x82, 0x43, 0xE4, 0x88, 0xA7, 0x43, + 0xE4, 0x8A, 0xA0, 0x43, 0xE4, 0x8C, 0x81, 0x43, + 0xE4, 0x8C, 0xB4, 0x43, 0xE4, 0x8D, 0x99, 0x43, + 0xE4, 0x8F, 0x95, 0x43, 0xE4, 0x8F, 0x99, 0x43, + 0xE4, 0x90, 0x8B, 0x43, 0xE4, 0x91, 0xAB, 0x43, + 0xE4, 0x94, 0xAB, 0x43, 0xE4, 0x95, 0x9D, 0x43, + 0xE4, 0x95, 0xA1, 0x43, 0xE4, 0x95, 0xAB, 0x43, + // Bytes 680 - 6bf + 0xE4, 0x97, 0x97, 0x43, 0xE4, 0x97, 0xB9, 0x43, + 0xE4, 0x98, 0xB5, 0x43, 0xE4, 0x9A, 0xBE, 0x43, + 0xE4, 0x9B, 0x87, 0x43, 0xE4, 0xA6, 0x95, 0x43, + 0xE4, 0xA7, 0xA6, 0x43, 0xE4, 0xA9, 0xAE, 0x43, + 0xE4, 0xA9, 0xB6, 0x43, 0xE4, 0xAA, 0xB2, 0x43, + 0xE4, 0xAC, 0xB3, 0x43, 0xE4, 0xAF, 0x8E, 0x43, + 0xE4, 0xB3, 0x8E, 0x43, 0xE4, 0xB3, 0xAD, 0x43, + 0xE4, 0xB3, 0xB8, 0x43, 0xE4, 0xB5, 0x96, 0x43, + // Bytes 6c0 - 6ff + 0xE4, 0xB8, 0x80, 0x43, 0xE4, 0xB8, 0x81, 0x43, + 0xE4, 0xB8, 0x83, 0x43, 0xE4, 0xB8, 0x89, 0x43, + 0xE4, 0xB8, 0x8A, 0x43, 0xE4, 0xB8, 0x8B, 0x43, + 0xE4, 0xB8, 0x8D, 0x43, 0xE4, 0xB8, 0x99, 0x43, + 0xE4, 0xB8, 0xA6, 0x43, 0xE4, 0xB8, 0xA8, 0x43, + 0xE4, 0xB8, 0xAD, 0x43, 0xE4, 0xB8, 0xB2, 0x43, + 0xE4, 0xB8, 0xB6, 0x43, 0xE4, 0xB8, 0xB8, 0x43, + 0xE4, 0xB8, 0xB9, 0x43, 0xE4, 0xB8, 0xBD, 0x43, + // Bytes 700 - 73f + 0xE4, 0xB8, 0xBF, 0x43, 0xE4, 0xB9, 0x81, 0x43, + 0xE4, 0xB9, 0x99, 0x43, 0xE4, 0xB9, 0x9D, 0x43, + 0xE4, 0xBA, 0x82, 0x43, 0xE4, 0xBA, 0x85, 0x43, + 0xE4, 0xBA, 0x86, 0x43, 0xE4, 0xBA, 0x8C, 0x43, + 0xE4, 0xBA, 0x94, 0x43, 0xE4, 0xBA, 0xA0, 0x43, + 0xE4, 0xBA, 0xA4, 0x43, 0xE4, 0xBA, 0xAE, 0x43, + 0xE4, 0xBA, 0xBA, 0x43, 0xE4, 0xBB, 0x80, 0x43, + 0xE4, 0xBB, 0x8C, 0x43, 0xE4, 0xBB, 0xA4, 0x43, + // Bytes 740 - 77f + 0xE4, 0xBC, 0x81, 0x43, 0xE4, 0xBC, 0x91, 0x43, + 0xE4, 0xBD, 0xA0, 0x43, 0xE4, 0xBE, 0x80, 0x43, + 0xE4, 0xBE, 0x86, 0x43, 0xE4, 0xBE, 0x8B, 0x43, + 0xE4, 0xBE, 0xAE, 0x43, 0xE4, 0xBE, 0xBB, 0x43, + 0xE4, 0xBE, 0xBF, 0x43, 0xE5, 0x80, 0x82, 0x43, + 0xE5, 0x80, 0xAB, 0x43, 0xE5, 0x81, 0xBA, 0x43, + 0xE5, 0x82, 0x99, 0x43, 0xE5, 0x83, 0x8F, 0x43, + 0xE5, 0x83, 0x9A, 0x43, 0xE5, 0x83, 0xA7, 0x43, + // Bytes 780 - 7bf + 0xE5, 0x84, 0xAA, 0x43, 0xE5, 0x84, 0xBF, 0x43, + 0xE5, 0x85, 0x80, 0x43, 0xE5, 0x85, 0x85, 0x43, + 0xE5, 0x85, 0x8D, 0x43, 0xE5, 0x85, 0x94, 0x43, + 0xE5, 0x85, 0xA4, 0x43, 0xE5, 0x85, 0xA5, 0x43, + 0xE5, 0x85, 0xA7, 0x43, 0xE5, 0x85, 0xA8, 0x43, + 0xE5, 0x85, 0xA9, 0x43, 0xE5, 0x85, 0xAB, 0x43, + 0xE5, 0x85, 0xAD, 0x43, 0xE5, 0x85, 0xB7, 0x43, + 0xE5, 0x86, 0x80, 0x43, 0xE5, 0x86, 0x82, 0x43, + // Bytes 7c0 - 7ff + 0xE5, 0x86, 0x8D, 0x43, 0xE5, 0x86, 0x92, 0x43, + 0xE5, 0x86, 0x95, 0x43, 0xE5, 0x86, 0x96, 0x43, + 0xE5, 0x86, 0x97, 0x43, 0xE5, 0x86, 0x99, 0x43, + 0xE5, 0x86, 0xA4, 0x43, 0xE5, 0x86, 0xAB, 0x43, + 0xE5, 0x86, 0xAC, 0x43, 0xE5, 0x86, 0xB5, 0x43, + 0xE5, 0x86, 0xB7, 0x43, 0xE5, 0x87, 0x89, 0x43, + 0xE5, 0x87, 0x8C, 0x43, 0xE5, 0x87, 0x9C, 0x43, + 0xE5, 0x87, 0x9E, 0x43, 0xE5, 0x87, 0xA0, 0x43, + // Bytes 800 - 83f + 0xE5, 0x87, 0xB5, 0x43, 0xE5, 0x88, 0x80, 0x43, + 0xE5, 0x88, 0x83, 0x43, 0xE5, 0x88, 0x87, 0x43, + 0xE5, 0x88, 0x97, 0x43, 0xE5, 0x88, 0x9D, 0x43, + 0xE5, 0x88, 0xA9, 0x43, 0xE5, 0x88, 0xBA, 0x43, + 0xE5, 0x88, 0xBB, 0x43, 0xE5, 0x89, 0x86, 0x43, + 0xE5, 0x89, 0x8D, 0x43, 0xE5, 0x89, 0xB2, 0x43, + 0xE5, 0x89, 0xB7, 0x43, 0xE5, 0x8A, 0x89, 0x43, + 0xE5, 0x8A, 0x9B, 0x43, 0xE5, 0x8A, 0xA3, 0x43, + // Bytes 840 - 87f + 0xE5, 0x8A, 0xB3, 0x43, 0xE5, 0x8A, 0xB4, 0x43, + 0xE5, 0x8B, 0x87, 0x43, 0xE5, 0x8B, 0x89, 0x43, + 0xE5, 0x8B, 0x92, 0x43, 0xE5, 0x8B, 0x9E, 0x43, + 0xE5, 0x8B, 0xA4, 0x43, 0xE5, 0x8B, 0xB5, 0x43, + 0xE5, 0x8B, 0xB9, 0x43, 0xE5, 0x8B, 0xBA, 0x43, + 0xE5, 0x8C, 0x85, 0x43, 0xE5, 0x8C, 0x86, 0x43, + 0xE5, 0x8C, 0x95, 0x43, 0xE5, 0x8C, 0x97, 0x43, + 0xE5, 0x8C, 0x9A, 0x43, 0xE5, 0x8C, 0xB8, 0x43, + // Bytes 880 - 8bf + 0xE5, 0x8C, 0xBB, 0x43, 0xE5, 0x8C, 0xBF, 0x43, + 0xE5, 0x8D, 0x81, 0x43, 0xE5, 0x8D, 0x84, 0x43, + 0xE5, 0x8D, 0x85, 0x43, 0xE5, 0x8D, 0x89, 0x43, + 0xE5, 0x8D, 0x91, 0x43, 0xE5, 0x8D, 0x94, 0x43, + 0xE5, 0x8D, 0x9A, 0x43, 0xE5, 0x8D, 0x9C, 0x43, + 0xE5, 0x8D, 0xA9, 0x43, 0xE5, 0x8D, 0xB0, 0x43, + 0xE5, 0x8D, 0xB3, 0x43, 0xE5, 0x8D, 0xB5, 0x43, + 0xE5, 0x8D, 0xBD, 0x43, 0xE5, 0x8D, 0xBF, 0x43, + // Bytes 8c0 - 8ff + 0xE5, 0x8E, 0x82, 0x43, 0xE5, 0x8E, 0xB6, 0x43, + 0xE5, 0x8F, 0x83, 0x43, 0xE5, 0x8F, 0x88, 0x43, + 0xE5, 0x8F, 0x8A, 0x43, 0xE5, 0x8F, 0x8C, 0x43, + 0xE5, 0x8F, 0x9F, 0x43, 0xE5, 0x8F, 0xA3, 0x43, + 0xE5, 0x8F, 0xA5, 0x43, 0xE5, 0x8F, 0xAB, 0x43, + 0xE5, 0x8F, 0xAF, 0x43, 0xE5, 0x8F, 0xB1, 0x43, + 0xE5, 0x8F, 0xB3, 0x43, 0xE5, 0x90, 0x86, 0x43, + 0xE5, 0x90, 0x88, 0x43, 0xE5, 0x90, 0x8D, 0x43, + // Bytes 900 - 93f + 0xE5, 0x90, 0x8F, 0x43, 0xE5, 0x90, 0x9D, 0x43, + 0xE5, 0x90, 0xB8, 0x43, 0xE5, 0x90, 0xB9, 0x43, + 0xE5, 0x91, 0x82, 0x43, 0xE5, 0x91, 0x88, 0x43, + 0xE5, 0x91, 0xA8, 0x43, 0xE5, 0x92, 0x9E, 0x43, + 0xE5, 0x92, 0xA2, 0x43, 0xE5, 0x92, 0xBD, 0x43, + 0xE5, 0x93, 0xB6, 0x43, 0xE5, 0x94, 0x90, 0x43, + 0xE5, 0x95, 0x8F, 0x43, 0xE5, 0x95, 0x93, 0x43, + 0xE5, 0x95, 0x95, 0x43, 0xE5, 0x95, 0xA3, 0x43, + // Bytes 940 - 97f + 0xE5, 0x96, 0x84, 0x43, 0xE5, 0x96, 0x87, 0x43, + 0xE5, 0x96, 0x99, 0x43, 0xE5, 0x96, 0x9D, 0x43, + 0xE5, 0x96, 0xAB, 0x43, 0xE5, 0x96, 0xB3, 0x43, + 0xE5, 0x96, 0xB6, 0x43, 0xE5, 0x97, 0x80, 0x43, + 0xE5, 0x97, 0x82, 0x43, 0xE5, 0x97, 0xA2, 0x43, + 0xE5, 0x98, 0x86, 0x43, 0xE5, 0x99, 0x91, 0x43, + 0xE5, 0x99, 0xA8, 0x43, 0xE5, 0x99, 0xB4, 0x43, + 0xE5, 0x9B, 0x97, 0x43, 0xE5, 0x9B, 0x9B, 0x43, + // Bytes 980 - 9bf + 0xE5, 0x9B, 0xB9, 0x43, 0xE5, 0x9C, 0x96, 0x43, + 0xE5, 0x9C, 0x97, 0x43, 0xE5, 0x9C, 0x9F, 0x43, + 0xE5, 0x9C, 0xB0, 0x43, 0xE5, 0x9E, 0x8B, 0x43, + 0xE5, 0x9F, 0x8E, 0x43, 0xE5, 0x9F, 0xB4, 0x43, + 0xE5, 0xA0, 0x8D, 0x43, 0xE5, 0xA0, 0xB1, 0x43, + 0xE5, 0xA0, 0xB2, 0x43, 0xE5, 0xA1, 0x80, 0x43, + 0xE5, 0xA1, 0x9A, 0x43, 0xE5, 0xA1, 0x9E, 0x43, + 0xE5, 0xA2, 0xA8, 0x43, 0xE5, 0xA2, 0xAC, 0x43, + // Bytes 9c0 - 9ff + 0xE5, 0xA2, 0xB3, 0x43, 0xE5, 0xA3, 0x98, 0x43, + 0xE5, 0xA3, 0x9F, 0x43, 0xE5, 0xA3, 0xAB, 0x43, + 0xE5, 0xA3, 0xAE, 0x43, 0xE5, 0xA3, 0xB0, 0x43, + 0xE5, 0xA3, 0xB2, 0x43, 0xE5, 0xA3, 0xB7, 0x43, + 0xE5, 0xA4, 0x82, 0x43, 0xE5, 0xA4, 0x86, 0x43, + 0xE5, 0xA4, 0x8A, 0x43, 0xE5, 0xA4, 0x95, 0x43, + 0xE5, 0xA4, 0x9A, 0x43, 0xE5, 0xA4, 0x9C, 0x43, + 0xE5, 0xA4, 0xA2, 0x43, 0xE5, 0xA4, 0xA7, 0x43, + // Bytes a00 - a3f + 0xE5, 0xA4, 0xA9, 0x43, 0xE5, 0xA5, 0x84, 0x43, + 0xE5, 0xA5, 0x88, 0x43, 0xE5, 0xA5, 0x91, 0x43, + 0xE5, 0xA5, 0x94, 0x43, 0xE5, 0xA5, 0xA2, 0x43, + 0xE5, 0xA5, 0xB3, 0x43, 0xE5, 0xA7, 0x98, 0x43, + 0xE5, 0xA7, 0xAC, 0x43, 0xE5, 0xA8, 0x9B, 0x43, + 0xE5, 0xA8, 0xA7, 0x43, 0xE5, 0xA9, 0xA2, 0x43, + 0xE5, 0xA9, 0xA6, 0x43, 0xE5, 0xAA, 0xB5, 0x43, + 0xE5, 0xAC, 0x88, 0x43, 0xE5, 0xAC, 0xA8, 0x43, + // Bytes a40 - a7f + 0xE5, 0xAC, 0xBE, 0x43, 0xE5, 0xAD, 0x90, 0x43, + 0xE5, 0xAD, 0x97, 0x43, 0xE5, 0xAD, 0xA6, 0x43, + 0xE5, 0xAE, 0x80, 0x43, 0xE5, 0xAE, 0x85, 0x43, + 0xE5, 0xAE, 0x97, 0x43, 0xE5, 0xAF, 0x83, 0x43, + 0xE5, 0xAF, 0x98, 0x43, 0xE5, 0xAF, 0xA7, 0x43, + 0xE5, 0xAF, 0xAE, 0x43, 0xE5, 0xAF, 0xB3, 0x43, + 0xE5, 0xAF, 0xB8, 0x43, 0xE5, 0xAF, 0xBF, 0x43, + 0xE5, 0xB0, 0x86, 0x43, 0xE5, 0xB0, 0x8F, 0x43, + // Bytes a80 - abf + 0xE5, 0xB0, 0xA2, 0x43, 0xE5, 0xB0, 0xB8, 0x43, + 0xE5, 0xB0, 0xBF, 0x43, 0xE5, 0xB1, 0xA0, 0x43, + 0xE5, 0xB1, 0xA2, 0x43, 0xE5, 0xB1, 0xA4, 0x43, + 0xE5, 0xB1, 0xA5, 0x43, 0xE5, 0xB1, 0xAE, 0x43, + 0xE5, 0xB1, 0xB1, 0x43, 0xE5, 0xB2, 0x8D, 0x43, + 0xE5, 0xB3, 0x80, 0x43, 0xE5, 0xB4, 0x99, 0x43, + 0xE5, 0xB5, 0x83, 0x43, 0xE5, 0xB5, 0x90, 0x43, + 0xE5, 0xB5, 0xAB, 0x43, 0xE5, 0xB5, 0xAE, 0x43, + // Bytes ac0 - aff + 0xE5, 0xB5, 0xBC, 0x43, 0xE5, 0xB6, 0xB2, 0x43, + 0xE5, 0xB6, 0xBA, 0x43, 0xE5, 0xB7, 0x9B, 0x43, + 0xE5, 0xB7, 0xA1, 0x43, 0xE5, 0xB7, 0xA2, 0x43, + 0xE5, 0xB7, 0xA5, 0x43, 0xE5, 0xB7, 0xA6, 0x43, + 0xE5, 0xB7, 0xB1, 0x43, 0xE5, 0xB7, 0xBD, 0x43, + 0xE5, 0xB7, 0xBE, 0x43, 0xE5, 0xB8, 0xA8, 0x43, + 0xE5, 0xB8, 0xBD, 0x43, 0xE5, 0xB9, 0xA9, 0x43, + 0xE5, 0xB9, 0xB2, 0x43, 0xE5, 0xB9, 0xB4, 0x43, + // Bytes b00 - b3f + 0xE5, 0xB9, 0xBA, 0x43, 0xE5, 0xB9, 0xBC, 0x43, + 0xE5, 0xB9, 0xBF, 0x43, 0xE5, 0xBA, 0xA6, 0x43, + 0xE5, 0xBA, 0xB0, 0x43, 0xE5, 0xBA, 0xB3, 0x43, + 0xE5, 0xBA, 0xB6, 0x43, 0xE5, 0xBB, 0x89, 0x43, + 0xE5, 0xBB, 0x8A, 0x43, 0xE5, 0xBB, 0x92, 0x43, + 0xE5, 0xBB, 0x93, 0x43, 0xE5, 0xBB, 0x99, 0x43, + 0xE5, 0xBB, 0xAC, 0x43, 0xE5, 0xBB, 0xB4, 0x43, + 0xE5, 0xBB, 0xBE, 0x43, 0xE5, 0xBC, 0x84, 0x43, + // Bytes b40 - b7f + 0xE5, 0xBC, 0x8B, 0x43, 0xE5, 0xBC, 0x93, 0x43, + 0xE5, 0xBC, 0xA2, 0x43, 0xE5, 0xBD, 0x90, 0x43, + 0xE5, 0xBD, 0x93, 0x43, 0xE5, 0xBD, 0xA1, 0x43, + 0xE5, 0xBD, 0xA2, 0x43, 0xE5, 0xBD, 0xA9, 0x43, + 0xE5, 0xBD, 0xAB, 0x43, 0xE5, 0xBD, 0xB3, 0x43, + 0xE5, 0xBE, 0x8B, 0x43, 0xE5, 0xBE, 0x8C, 0x43, + 0xE5, 0xBE, 0x97, 0x43, 0xE5, 0xBE, 0x9A, 0x43, + 0xE5, 0xBE, 0xA9, 0x43, 0xE5, 0xBE, 0xAD, 0x43, + // Bytes b80 - bbf + 0xE5, 0xBF, 0x83, 0x43, 0xE5, 0xBF, 0x8D, 0x43, + 0xE5, 0xBF, 0x97, 0x43, 0xE5, 0xBF, 0xB5, 0x43, + 0xE5, 0xBF, 0xB9, 0x43, 0xE6, 0x80, 0x92, 0x43, + 0xE6, 0x80, 0x9C, 0x43, 0xE6, 0x81, 0xB5, 0x43, + 0xE6, 0x82, 0x81, 0x43, 0xE6, 0x82, 0x94, 0x43, + 0xE6, 0x83, 0x87, 0x43, 0xE6, 0x83, 0x98, 0x43, + 0xE6, 0x83, 0xA1, 0x43, 0xE6, 0x84, 0x88, 0x43, + 0xE6, 0x85, 0x84, 0x43, 0xE6, 0x85, 0x88, 0x43, + // Bytes bc0 - bff + 0xE6, 0x85, 0x8C, 0x43, 0xE6, 0x85, 0x8E, 0x43, + 0xE6, 0x85, 0xA0, 0x43, 0xE6, 0x85, 0xA8, 0x43, + 0xE6, 0x85, 0xBA, 0x43, 0xE6, 0x86, 0x8E, 0x43, + 0xE6, 0x86, 0x90, 0x43, 0xE6, 0x86, 0xA4, 0x43, + 0xE6, 0x86, 0xAF, 0x43, 0xE6, 0x86, 0xB2, 0x43, + 0xE6, 0x87, 0x9E, 0x43, 0xE6, 0x87, 0xB2, 0x43, + 0xE6, 0x87, 0xB6, 0x43, 0xE6, 0x88, 0x80, 0x43, + 0xE6, 0x88, 0x88, 0x43, 0xE6, 0x88, 0x90, 0x43, + // Bytes c00 - c3f + 0xE6, 0x88, 0x9B, 0x43, 0xE6, 0x88, 0xAE, 0x43, + 0xE6, 0x88, 0xB4, 0x43, 0xE6, 0x88, 0xB6, 0x43, + 0xE6, 0x89, 0x8B, 0x43, 0xE6, 0x89, 0x93, 0x43, + 0xE6, 0x89, 0x9D, 0x43, 0xE6, 0x8A, 0x95, 0x43, + 0xE6, 0x8A, 0xB1, 0x43, 0xE6, 0x8B, 0x89, 0x43, + 0xE6, 0x8B, 0x8F, 0x43, 0xE6, 0x8B, 0x93, 0x43, + 0xE6, 0x8B, 0x94, 0x43, 0xE6, 0x8B, 0xBC, 0x43, + 0xE6, 0x8B, 0xBE, 0x43, 0xE6, 0x8C, 0x87, 0x43, + // Bytes c40 - c7f + 0xE6, 0x8C, 0xBD, 0x43, 0xE6, 0x8D, 0x90, 0x43, + 0xE6, 0x8D, 0x95, 0x43, 0xE6, 0x8D, 0xA8, 0x43, + 0xE6, 0x8D, 0xBB, 0x43, 0xE6, 0x8E, 0x83, 0x43, + 0xE6, 0x8E, 0xA0, 0x43, 0xE6, 0x8E, 0xA9, 0x43, + 0xE6, 0x8F, 0x84, 0x43, 0xE6, 0x8F, 0x85, 0x43, + 0xE6, 0x8F, 0xA4, 0x43, 0xE6, 0x90, 0x9C, 0x43, + 0xE6, 0x90, 0xA2, 0x43, 0xE6, 0x91, 0x92, 0x43, + 0xE6, 0x91, 0xA9, 0x43, 0xE6, 0x91, 0xB7, 0x43, + // Bytes c80 - cbf + 0xE6, 0x91, 0xBE, 0x43, 0xE6, 0x92, 0x9A, 0x43, + 0xE6, 0x92, 0x9D, 0x43, 0xE6, 0x93, 0x84, 0x43, + 0xE6, 0x94, 0xAF, 0x43, 0xE6, 0x94, 0xB4, 0x43, + 0xE6, 0x95, 0x8F, 0x43, 0xE6, 0x95, 0x96, 0x43, + 0xE6, 0x95, 0xAC, 0x43, 0xE6, 0x95, 0xB8, 0x43, + 0xE6, 0x96, 0x87, 0x43, 0xE6, 0x96, 0x97, 0x43, + 0xE6, 0x96, 0x99, 0x43, 0xE6, 0x96, 0xA4, 0x43, + 0xE6, 0x96, 0xB0, 0x43, 0xE6, 0x96, 0xB9, 0x43, + // Bytes cc0 - cff + 0xE6, 0x97, 0x85, 0x43, 0xE6, 0x97, 0xA0, 0x43, + 0xE6, 0x97, 0xA2, 0x43, 0xE6, 0x97, 0xA3, 0x43, + 0xE6, 0x97, 0xA5, 0x43, 0xE6, 0x98, 0x93, 0x43, + 0xE6, 0x98, 0xA0, 0x43, 0xE6, 0x99, 0x89, 0x43, + 0xE6, 0x99, 0xB4, 0x43, 0xE6, 0x9A, 0x88, 0x43, + 0xE6, 0x9A, 0x91, 0x43, 0xE6, 0x9A, 0x9C, 0x43, + 0xE6, 0x9A, 0xB4, 0x43, 0xE6, 0x9B, 0x86, 0x43, + 0xE6, 0x9B, 0xB0, 0x43, 0xE6, 0x9B, 0xB4, 0x43, + // Bytes d00 - d3f + 0xE6, 0x9B, 0xB8, 0x43, 0xE6, 0x9C, 0x80, 0x43, + 0xE6, 0x9C, 0x88, 0x43, 0xE6, 0x9C, 0x89, 0x43, + 0xE6, 0x9C, 0x97, 0x43, 0xE6, 0x9C, 0x9B, 0x43, + 0xE6, 0x9C, 0xA1, 0x43, 0xE6, 0x9C, 0xA8, 0x43, + 0xE6, 0x9D, 0x8E, 0x43, 0xE6, 0x9D, 0x93, 0x43, + 0xE6, 0x9D, 0x96, 0x43, 0xE6, 0x9D, 0x9E, 0x43, + 0xE6, 0x9D, 0xBB, 0x43, 0xE6, 0x9E, 0x85, 0x43, + 0xE6, 0x9E, 0x97, 0x43, 0xE6, 0x9F, 0xB3, 0x43, + // Bytes d40 - d7f + 0xE6, 0x9F, 0xBA, 0x43, 0xE6, 0xA0, 0x97, 0x43, + 0xE6, 0xA0, 0x9F, 0x43, 0xE6, 0xA0, 0xAA, 0x43, + 0xE6, 0xA1, 0x92, 0x43, 0xE6, 0xA2, 0x81, 0x43, + 0xE6, 0xA2, 0x85, 0x43, 0xE6, 0xA2, 0x8E, 0x43, + 0xE6, 0xA2, 0xA8, 0x43, 0xE6, 0xA4, 0x94, 0x43, + 0xE6, 0xA5, 0x82, 0x43, 0xE6, 0xA6, 0xA3, 0x43, + 0xE6, 0xA7, 0xAA, 0x43, 0xE6, 0xA8, 0x82, 0x43, + 0xE6, 0xA8, 0x93, 0x43, 0xE6, 0xAA, 0xA8, 0x43, + // Bytes d80 - dbf + 0xE6, 0xAB, 0x93, 0x43, 0xE6, 0xAB, 0x9B, 0x43, + 0xE6, 0xAC, 0x84, 0x43, 0xE6, 0xAC, 0xA0, 0x43, + 0xE6, 0xAC, 0xA1, 0x43, 0xE6, 0xAD, 0x94, 0x43, + 0xE6, 0xAD, 0xA2, 0x43, 0xE6, 0xAD, 0xA3, 0x43, + 0xE6, 0xAD, 0xB2, 0x43, 0xE6, 0xAD, 0xB7, 0x43, + 0xE6, 0xAD, 0xB9, 0x43, 0xE6, 0xAE, 0x9F, 0x43, + 0xE6, 0xAE, 0xAE, 0x43, 0xE6, 0xAE, 0xB3, 0x43, + 0xE6, 0xAE, 0xBA, 0x43, 0xE6, 0xAE, 0xBB, 0x43, + // Bytes dc0 - dff + 0xE6, 0xAF, 0x8B, 0x43, 0xE6, 0xAF, 0x8D, 0x43, + 0xE6, 0xAF, 0x94, 0x43, 0xE6, 0xAF, 0x9B, 0x43, + 0xE6, 0xB0, 0x8F, 0x43, 0xE6, 0xB0, 0x94, 0x43, + 0xE6, 0xB0, 0xB4, 0x43, 0xE6, 0xB1, 0x8E, 0x43, + 0xE6, 0xB1, 0xA7, 0x43, 0xE6, 0xB2, 0x88, 0x43, + 0xE6, 0xB2, 0xBF, 0x43, 0xE6, 0xB3, 0x8C, 0x43, + 0xE6, 0xB3, 0x8D, 0x43, 0xE6, 0xB3, 0xA5, 0x43, + 0xE6, 0xB3, 0xA8, 0x43, 0xE6, 0xB4, 0x96, 0x43, + // Bytes e00 - e3f + 0xE6, 0xB4, 0x9B, 0x43, 0xE6, 0xB4, 0x9E, 0x43, + 0xE6, 0xB4, 0xB4, 0x43, 0xE6, 0xB4, 0xBE, 0x43, + 0xE6, 0xB5, 0x81, 0x43, 0xE6, 0xB5, 0xA9, 0x43, + 0xE6, 0xB5, 0xAA, 0x43, 0xE6, 0xB5, 0xB7, 0x43, + 0xE6, 0xB5, 0xB8, 0x43, 0xE6, 0xB6, 0x85, 0x43, + 0xE6, 0xB7, 0x8B, 0x43, 0xE6, 0xB7, 0x9A, 0x43, + 0xE6, 0xB7, 0xAA, 0x43, 0xE6, 0xB7, 0xB9, 0x43, + 0xE6, 0xB8, 0x9A, 0x43, 0xE6, 0xB8, 0xAF, 0x43, + // Bytes e40 - e7f + 0xE6, 0xB9, 0xAE, 0x43, 0xE6, 0xBA, 0x80, 0x43, + 0xE6, 0xBA, 0x9C, 0x43, 0xE6, 0xBA, 0xBA, 0x43, + 0xE6, 0xBB, 0x87, 0x43, 0xE6, 0xBB, 0x8B, 0x43, + 0xE6, 0xBB, 0x91, 0x43, 0xE6, 0xBB, 0x9B, 0x43, + 0xE6, 0xBC, 0x8F, 0x43, 0xE6, 0xBC, 0x94, 0x43, + 0xE6, 0xBC, 0xA2, 0x43, 0xE6, 0xBC, 0xA3, 0x43, + 0xE6, 0xBD, 0xAE, 0x43, 0xE6, 0xBF, 0x86, 0x43, + 0xE6, 0xBF, 0xAB, 0x43, 0xE6, 0xBF, 0xBE, 0x43, + // Bytes e80 - ebf + 0xE7, 0x80, 0x9B, 0x43, 0xE7, 0x80, 0x9E, 0x43, + 0xE7, 0x80, 0xB9, 0x43, 0xE7, 0x81, 0x8A, 0x43, + 0xE7, 0x81, 0xAB, 0x43, 0xE7, 0x81, 0xB0, 0x43, + 0xE7, 0x81, 0xB7, 0x43, 0xE7, 0x81, 0xBD, 0x43, + 0xE7, 0x82, 0x99, 0x43, 0xE7, 0x82, 0xAD, 0x43, + 0xE7, 0x83, 0x88, 0x43, 0xE7, 0x83, 0x99, 0x43, + 0xE7, 0x84, 0xA1, 0x43, 0xE7, 0x85, 0x85, 0x43, + 0xE7, 0x85, 0x89, 0x43, 0xE7, 0x85, 0xAE, 0x43, + // Bytes ec0 - eff + 0xE7, 0x86, 0x9C, 0x43, 0xE7, 0x87, 0x8E, 0x43, + 0xE7, 0x87, 0x90, 0x43, 0xE7, 0x88, 0x90, 0x43, + 0xE7, 0x88, 0x9B, 0x43, 0xE7, 0x88, 0xA8, 0x43, + 0xE7, 0x88, 0xAA, 0x43, 0xE7, 0x88, 0xAB, 0x43, + 0xE7, 0x88, 0xB5, 0x43, 0xE7, 0x88, 0xB6, 0x43, + 0xE7, 0x88, 0xBB, 0x43, 0xE7, 0x88, 0xBF, 0x43, + 0xE7, 0x89, 0x87, 0x43, 0xE7, 0x89, 0x90, 0x43, + 0xE7, 0x89, 0x99, 0x43, 0xE7, 0x89, 0x9B, 0x43, + // Bytes f00 - f3f + 0xE7, 0x89, 0xA2, 0x43, 0xE7, 0x89, 0xB9, 0x43, + 0xE7, 0x8A, 0x80, 0x43, 0xE7, 0x8A, 0x95, 0x43, + 0xE7, 0x8A, 0xAC, 0x43, 0xE7, 0x8A, 0xAF, 0x43, + 0xE7, 0x8B, 0x80, 0x43, 0xE7, 0x8B, 0xBC, 0x43, + 0xE7, 0x8C, 0xAA, 0x43, 0xE7, 0x8D, 0xB5, 0x43, + 0xE7, 0x8D, 0xBA, 0x43, 0xE7, 0x8E, 0x84, 0x43, + 0xE7, 0x8E, 0x87, 0x43, 0xE7, 0x8E, 0x89, 0x43, + 0xE7, 0x8E, 0x8B, 0x43, 0xE7, 0x8E, 0xA5, 0x43, + // Bytes f40 - f7f + 0xE7, 0x8E, 0xB2, 0x43, 0xE7, 0x8F, 0x9E, 0x43, + 0xE7, 0x90, 0x86, 0x43, 0xE7, 0x90, 0x89, 0x43, + 0xE7, 0x90, 0xA2, 0x43, 0xE7, 0x91, 0x87, 0x43, + 0xE7, 0x91, 0x9C, 0x43, 0xE7, 0x91, 0xA9, 0x43, + 0xE7, 0x91, 0xB1, 0x43, 0xE7, 0x92, 0x85, 0x43, + 0xE7, 0x92, 0x89, 0x43, 0xE7, 0x92, 0x98, 0x43, + 0xE7, 0x93, 0x8A, 0x43, 0xE7, 0x93, 0x9C, 0x43, + 0xE7, 0x93, 0xA6, 0x43, 0xE7, 0x94, 0x86, 0x43, + // Bytes f80 - fbf + 0xE7, 0x94, 0x98, 0x43, 0xE7, 0x94, 0x9F, 0x43, + 0xE7, 0x94, 0xA4, 0x43, 0xE7, 0x94, 0xA8, 0x43, + 0xE7, 0x94, 0xB0, 0x43, 0xE7, 0x94, 0xB2, 0x43, + 0xE7, 0x94, 0xB3, 0x43, 0xE7, 0x94, 0xB7, 0x43, + 0xE7, 0x94, 0xBB, 0x43, 0xE7, 0x94, 0xBE, 0x43, + 0xE7, 0x95, 0x99, 0x43, 0xE7, 0x95, 0xA5, 0x43, + 0xE7, 0x95, 0xB0, 0x43, 0xE7, 0x96, 0x8B, 0x43, + 0xE7, 0x96, 0x92, 0x43, 0xE7, 0x97, 0xA2, 0x43, + // Bytes fc0 - fff + 0xE7, 0x98, 0x90, 0x43, 0xE7, 0x98, 0x9D, 0x43, + 0xE7, 0x98, 0x9F, 0x43, 0xE7, 0x99, 0x82, 0x43, + 0xE7, 0x99, 0xA9, 0x43, 0xE7, 0x99, 0xB6, 0x43, + 0xE7, 0x99, 0xBD, 0x43, 0xE7, 0x9A, 0xAE, 0x43, + 0xE7, 0x9A, 0xBF, 0x43, 0xE7, 0x9B, 0x8A, 0x43, + 0xE7, 0x9B, 0x9B, 0x43, 0xE7, 0x9B, 0xA3, 0x43, + 0xE7, 0x9B, 0xA7, 0x43, 0xE7, 0x9B, 0xAE, 0x43, + 0xE7, 0x9B, 0xB4, 0x43, 0xE7, 0x9C, 0x81, 0x43, + // Bytes 1000 - 103f + 0xE7, 0x9C, 0x9E, 0x43, 0xE7, 0x9C, 0x9F, 0x43, + 0xE7, 0x9D, 0x80, 0x43, 0xE7, 0x9D, 0x8A, 0x43, + 0xE7, 0x9E, 0x8B, 0x43, 0xE7, 0x9E, 0xA7, 0x43, + 0xE7, 0x9F, 0x9B, 0x43, 0xE7, 0x9F, 0xA2, 0x43, + 0xE7, 0x9F, 0xB3, 0x43, 0xE7, 0xA1, 0x8E, 0x43, + 0xE7, 0xA1, 0xAB, 0x43, 0xE7, 0xA2, 0x8C, 0x43, + 0xE7, 0xA2, 0x91, 0x43, 0xE7, 0xA3, 0x8A, 0x43, + 0xE7, 0xA3, 0x8C, 0x43, 0xE7, 0xA3, 0xBB, 0x43, + // Bytes 1040 - 107f + 0xE7, 0xA4, 0xAA, 0x43, 0xE7, 0xA4, 0xBA, 0x43, + 0xE7, 0xA4, 0xBC, 0x43, 0xE7, 0xA4, 0xBE, 0x43, + 0xE7, 0xA5, 0x88, 0x43, 0xE7, 0xA5, 0x89, 0x43, + 0xE7, 0xA5, 0x90, 0x43, 0xE7, 0xA5, 0x96, 0x43, + 0xE7, 0xA5, 0x9D, 0x43, 0xE7, 0xA5, 0x9E, 0x43, + 0xE7, 0xA5, 0xA5, 0x43, 0xE7, 0xA5, 0xBF, 0x43, + 0xE7, 0xA6, 0x81, 0x43, 0xE7, 0xA6, 0x8D, 0x43, + 0xE7, 0xA6, 0x8E, 0x43, 0xE7, 0xA6, 0x8F, 0x43, + // Bytes 1080 - 10bf + 0xE7, 0xA6, 0xAE, 0x43, 0xE7, 0xA6, 0xB8, 0x43, + 0xE7, 0xA6, 0xBE, 0x43, 0xE7, 0xA7, 0x8A, 0x43, + 0xE7, 0xA7, 0x98, 0x43, 0xE7, 0xA7, 0xAB, 0x43, + 0xE7, 0xA8, 0x9C, 0x43, 0xE7, 0xA9, 0x80, 0x43, + 0xE7, 0xA9, 0x8A, 0x43, 0xE7, 0xA9, 0x8F, 0x43, + 0xE7, 0xA9, 0xB4, 0x43, 0xE7, 0xA9, 0xBA, 0x43, + 0xE7, 0xAA, 0x81, 0x43, 0xE7, 0xAA, 0xB1, 0x43, + 0xE7, 0xAB, 0x8B, 0x43, 0xE7, 0xAB, 0xAE, 0x43, + // Bytes 10c0 - 10ff + 0xE7, 0xAB, 0xB9, 0x43, 0xE7, 0xAC, 0xA0, 0x43, + 0xE7, 0xAE, 0x8F, 0x43, 0xE7, 0xAF, 0x80, 0x43, + 0xE7, 0xAF, 0x86, 0x43, 0xE7, 0xAF, 0x89, 0x43, + 0xE7, 0xB0, 0xBE, 0x43, 0xE7, 0xB1, 0xA0, 0x43, + 0xE7, 0xB1, 0xB3, 0x43, 0xE7, 0xB1, 0xBB, 0x43, + 0xE7, 0xB2, 0x92, 0x43, 0xE7, 0xB2, 0xBE, 0x43, + 0xE7, 0xB3, 0x92, 0x43, 0xE7, 0xB3, 0x96, 0x43, + 0xE7, 0xB3, 0xA3, 0x43, 0xE7, 0xB3, 0xA7, 0x43, + // Bytes 1100 - 113f + 0xE7, 0xB3, 0xA8, 0x43, 0xE7, 0xB3, 0xB8, 0x43, + 0xE7, 0xB4, 0x80, 0x43, 0xE7, 0xB4, 0x90, 0x43, + 0xE7, 0xB4, 0xA2, 0x43, 0xE7, 0xB4, 0xAF, 0x43, + 0xE7, 0xB5, 0x82, 0x43, 0xE7, 0xB5, 0x9B, 0x43, + 0xE7, 0xB5, 0xA3, 0x43, 0xE7, 0xB6, 0xA0, 0x43, + 0xE7, 0xB6, 0xBE, 0x43, 0xE7, 0xB7, 0x87, 0x43, + 0xE7, 0xB7, 0xB4, 0x43, 0xE7, 0xB8, 0x82, 0x43, + 0xE7, 0xB8, 0x89, 0x43, 0xE7, 0xB8, 0xB7, 0x43, + // Bytes 1140 - 117f + 0xE7, 0xB9, 0x81, 0x43, 0xE7, 0xB9, 0x85, 0x43, + 0xE7, 0xBC, 0xB6, 0x43, 0xE7, 0xBC, 0xBE, 0x43, + 0xE7, 0xBD, 0x91, 0x43, 0xE7, 0xBD, 0xB2, 0x43, + 0xE7, 0xBD, 0xB9, 0x43, 0xE7, 0xBD, 0xBA, 0x43, + 0xE7, 0xBE, 0x85, 0x43, 0xE7, 0xBE, 0x8A, 0x43, + 0xE7, 0xBE, 0x95, 0x43, 0xE7, 0xBE, 0x9A, 0x43, + 0xE7, 0xBE, 0xBD, 0x43, 0xE7, 0xBF, 0xBA, 0x43, + 0xE8, 0x80, 0x81, 0x43, 0xE8, 0x80, 0x85, 0x43, + // Bytes 1180 - 11bf + 0xE8, 0x80, 0x8C, 0x43, 0xE8, 0x80, 0x92, 0x43, + 0xE8, 0x80, 0xB3, 0x43, 0xE8, 0x81, 0x86, 0x43, + 0xE8, 0x81, 0xA0, 0x43, 0xE8, 0x81, 0xAF, 0x43, + 0xE8, 0x81, 0xB0, 0x43, 0xE8, 0x81, 0xBE, 0x43, + 0xE8, 0x81, 0xBF, 0x43, 0xE8, 0x82, 0x89, 0x43, + 0xE8, 0x82, 0x8B, 0x43, 0xE8, 0x82, 0xAD, 0x43, + 0xE8, 0x82, 0xB2, 0x43, 0xE8, 0x84, 0x83, 0x43, + 0xE8, 0x84, 0xBE, 0x43, 0xE8, 0x87, 0x98, 0x43, + // Bytes 11c0 - 11ff + 0xE8, 0x87, 0xA3, 0x43, 0xE8, 0x87, 0xA8, 0x43, + 0xE8, 0x87, 0xAA, 0x43, 0xE8, 0x87, 0xAD, 0x43, + 0xE8, 0x87, 0xB3, 0x43, 0xE8, 0x87, 0xBC, 0x43, + 0xE8, 0x88, 0x81, 0x43, 0xE8, 0x88, 0x84, 0x43, + 0xE8, 0x88, 0x8C, 0x43, 0xE8, 0x88, 0x98, 0x43, + 0xE8, 0x88, 0x9B, 0x43, 0xE8, 0x88, 0x9F, 0x43, + 0xE8, 0x89, 0xAE, 0x43, 0xE8, 0x89, 0xAF, 0x43, + 0xE8, 0x89, 0xB2, 0x43, 0xE8, 0x89, 0xB8, 0x43, + // Bytes 1200 - 123f + 0xE8, 0x89, 0xB9, 0x43, 0xE8, 0x8A, 0x8B, 0x43, + 0xE8, 0x8A, 0x91, 0x43, 0xE8, 0x8A, 0x9D, 0x43, + 0xE8, 0x8A, 0xB1, 0x43, 0xE8, 0x8A, 0xB3, 0x43, + 0xE8, 0x8A, 0xBD, 0x43, 0xE8, 0x8B, 0xA5, 0x43, + 0xE8, 0x8B, 0xA6, 0x43, 0xE8, 0x8C, 0x9D, 0x43, + 0xE8, 0x8C, 0xA3, 0x43, 0xE8, 0x8C, 0xB6, 0x43, + 0xE8, 0x8D, 0x92, 0x43, 0xE8, 0x8D, 0x93, 0x43, + 0xE8, 0x8D, 0xA3, 0x43, 0xE8, 0x8E, 0xAD, 0x43, + // Bytes 1240 - 127f + 0xE8, 0x8E, 0xBD, 0x43, 0xE8, 0x8F, 0x89, 0x43, + 0xE8, 0x8F, 0x8A, 0x43, 0xE8, 0x8F, 0x8C, 0x43, + 0xE8, 0x8F, 0x9C, 0x43, 0xE8, 0x8F, 0xA7, 0x43, + 0xE8, 0x8F, 0xAF, 0x43, 0xE8, 0x8F, 0xB1, 0x43, + 0xE8, 0x90, 0xBD, 0x43, 0xE8, 0x91, 0x89, 0x43, + 0xE8, 0x91, 0x97, 0x43, 0xE8, 0x93, 0xAE, 0x43, + 0xE8, 0x93, 0xB1, 0x43, 0xE8, 0x93, 0xB3, 0x43, + 0xE8, 0x93, 0xBC, 0x43, 0xE8, 0x94, 0x96, 0x43, + // Bytes 1280 - 12bf + 0xE8, 0x95, 0xA4, 0x43, 0xE8, 0x97, 0x8D, 0x43, + 0xE8, 0x97, 0xBA, 0x43, 0xE8, 0x98, 0x86, 0x43, + 0xE8, 0x98, 0x92, 0x43, 0xE8, 0x98, 0xAD, 0x43, + 0xE8, 0x98, 0xBF, 0x43, 0xE8, 0x99, 0x8D, 0x43, + 0xE8, 0x99, 0x90, 0x43, 0xE8, 0x99, 0x9C, 0x43, + 0xE8, 0x99, 0xA7, 0x43, 0xE8, 0x99, 0xA9, 0x43, + 0xE8, 0x99, 0xAB, 0x43, 0xE8, 0x9A, 0x88, 0x43, + 0xE8, 0x9A, 0xA9, 0x43, 0xE8, 0x9B, 0xA2, 0x43, + // Bytes 12c0 - 12ff + 0xE8, 0x9C, 0x8E, 0x43, 0xE8, 0x9C, 0xA8, 0x43, + 0xE8, 0x9D, 0xAB, 0x43, 0xE8, 0x9D, 0xB9, 0x43, + 0xE8, 0x9E, 0x86, 0x43, 0xE8, 0x9E, 0xBA, 0x43, + 0xE8, 0x9F, 0xA1, 0x43, 0xE8, 0xA0, 0x81, 0x43, + 0xE8, 0xA0, 0x9F, 0x43, 0xE8, 0xA1, 0x80, 0x43, + 0xE8, 0xA1, 0x8C, 0x43, 0xE8, 0xA1, 0xA0, 0x43, + 0xE8, 0xA1, 0xA3, 0x43, 0xE8, 0xA3, 0x82, 0x43, + 0xE8, 0xA3, 0x8F, 0x43, 0xE8, 0xA3, 0x97, 0x43, + // Bytes 1300 - 133f + 0xE8, 0xA3, 0x9E, 0x43, 0xE8, 0xA3, 0xA1, 0x43, + 0xE8, 0xA3, 0xB8, 0x43, 0xE8, 0xA3, 0xBA, 0x43, + 0xE8, 0xA4, 0x90, 0x43, 0xE8, 0xA5, 0x81, 0x43, + 0xE8, 0xA5, 0xA4, 0x43, 0xE8, 0xA5, 0xBE, 0x43, + 0xE8, 0xA6, 0x86, 0x43, 0xE8, 0xA6, 0x8B, 0x43, + 0xE8, 0xA6, 0x96, 0x43, 0xE8, 0xA7, 0x92, 0x43, + 0xE8, 0xA7, 0xA3, 0x43, 0xE8, 0xA8, 0x80, 0x43, + 0xE8, 0xAA, 0xA0, 0x43, 0xE8, 0xAA, 0xAA, 0x43, + // Bytes 1340 - 137f + 0xE8, 0xAA, 0xBF, 0x43, 0xE8, 0xAB, 0x8B, 0x43, + 0xE8, 0xAB, 0x92, 0x43, 0xE8, 0xAB, 0x96, 0x43, + 0xE8, 0xAB, 0xAD, 0x43, 0xE8, 0xAB, 0xB8, 0x43, + 0xE8, 0xAB, 0xBE, 0x43, 0xE8, 0xAC, 0x81, 0x43, + 0xE8, 0xAC, 0xB9, 0x43, 0xE8, 0xAD, 0x98, 0x43, + 0xE8, 0xAE, 0x80, 0x43, 0xE8, 0xAE, 0x8A, 0x43, + 0xE8, 0xB0, 0xB7, 0x43, 0xE8, 0xB1, 0x86, 0x43, + 0xE8, 0xB1, 0x88, 0x43, 0xE8, 0xB1, 0x95, 0x43, + // Bytes 1380 - 13bf + 0xE8, 0xB1, 0xB8, 0x43, 0xE8, 0xB2, 0x9D, 0x43, + 0xE8, 0xB2, 0xA1, 0x43, 0xE8, 0xB2, 0xA9, 0x43, + 0xE8, 0xB2, 0xAB, 0x43, 0xE8, 0xB3, 0x81, 0x43, + 0xE8, 0xB3, 0x82, 0x43, 0xE8, 0xB3, 0x87, 0x43, + 0xE8, 0xB3, 0x88, 0x43, 0xE8, 0xB3, 0x93, 0x43, + 0xE8, 0xB4, 0x88, 0x43, 0xE8, 0xB4, 0x9B, 0x43, + 0xE8, 0xB5, 0xA4, 0x43, 0xE8, 0xB5, 0xB0, 0x43, + 0xE8, 0xB5, 0xB7, 0x43, 0xE8, 0xB6, 0xB3, 0x43, + // Bytes 13c0 - 13ff + 0xE8, 0xB6, 0xBC, 0x43, 0xE8, 0xB7, 0x8B, 0x43, + 0xE8, 0xB7, 0xAF, 0x43, 0xE8, 0xB7, 0xB0, 0x43, + 0xE8, 0xBA, 0xAB, 0x43, 0xE8, 0xBB, 0x8A, 0x43, + 0xE8, 0xBB, 0x94, 0x43, 0xE8, 0xBC, 0xA6, 0x43, + 0xE8, 0xBC, 0xAA, 0x43, 0xE8, 0xBC, 0xB8, 0x43, + 0xE8, 0xBC, 0xBB, 0x43, 0xE8, 0xBD, 0xA2, 0x43, + 0xE8, 0xBE, 0x9B, 0x43, 0xE8, 0xBE, 0x9E, 0x43, + 0xE8, 0xBE, 0xB0, 0x43, 0xE8, 0xBE, 0xB5, 0x43, + // Bytes 1400 - 143f + 0xE8, 0xBE, 0xB6, 0x43, 0xE9, 0x80, 0xA3, 0x43, + 0xE9, 0x80, 0xB8, 0x43, 0xE9, 0x81, 0x8A, 0x43, + 0xE9, 0x81, 0xA9, 0x43, 0xE9, 0x81, 0xB2, 0x43, + 0xE9, 0x81, 0xBC, 0x43, 0xE9, 0x82, 0x8F, 0x43, + 0xE9, 0x82, 0x91, 0x43, 0xE9, 0x82, 0x94, 0x43, + 0xE9, 0x83, 0x8E, 0x43, 0xE9, 0x83, 0x9E, 0x43, + 0xE9, 0x83, 0xB1, 0x43, 0xE9, 0x83, 0xBD, 0x43, + 0xE9, 0x84, 0x91, 0x43, 0xE9, 0x84, 0x9B, 0x43, + // Bytes 1440 - 147f + 0xE9, 0x85, 0x89, 0x43, 0xE9, 0x85, 0x8D, 0x43, + 0xE9, 0x85, 0xAA, 0x43, 0xE9, 0x86, 0x99, 0x43, + 0xE9, 0x86, 0xB4, 0x43, 0xE9, 0x87, 0x86, 0x43, + 0xE9, 0x87, 0x8C, 0x43, 0xE9, 0x87, 0x8F, 0x43, + 0xE9, 0x87, 0x91, 0x43, 0xE9, 0x88, 0xB4, 0x43, + 0xE9, 0x88, 0xB8, 0x43, 0xE9, 0x89, 0xB6, 0x43, + 0xE9, 0x89, 0xBC, 0x43, 0xE9, 0x8B, 0x97, 0x43, + 0xE9, 0x8B, 0x98, 0x43, 0xE9, 0x8C, 0x84, 0x43, + // Bytes 1480 - 14bf + 0xE9, 0x8D, 0x8A, 0x43, 0xE9, 0x8F, 0xB9, 0x43, + 0xE9, 0x90, 0x95, 0x43, 0xE9, 0x95, 0xB7, 0x43, + 0xE9, 0x96, 0x80, 0x43, 0xE9, 0x96, 0x8B, 0x43, + 0xE9, 0x96, 0xAD, 0x43, 0xE9, 0x96, 0xB7, 0x43, + 0xE9, 0x98, 0x9C, 0x43, 0xE9, 0x98, 0xAE, 0x43, + 0xE9, 0x99, 0x8B, 0x43, 0xE9, 0x99, 0x8D, 0x43, + 0xE9, 0x99, 0xB5, 0x43, 0xE9, 0x99, 0xB8, 0x43, + 0xE9, 0x99, 0xBC, 0x43, 0xE9, 0x9A, 0x86, 0x43, + // Bytes 14c0 - 14ff + 0xE9, 0x9A, 0xA3, 0x43, 0xE9, 0x9A, 0xB6, 0x43, + 0xE9, 0x9A, 0xB7, 0x43, 0xE9, 0x9A, 0xB8, 0x43, + 0xE9, 0x9A, 0xB9, 0x43, 0xE9, 0x9B, 0x83, 0x43, + 0xE9, 0x9B, 0xA2, 0x43, 0xE9, 0x9B, 0xA3, 0x43, + 0xE9, 0x9B, 0xA8, 0x43, 0xE9, 0x9B, 0xB6, 0x43, + 0xE9, 0x9B, 0xB7, 0x43, 0xE9, 0x9C, 0xA3, 0x43, + 0xE9, 0x9C, 0xB2, 0x43, 0xE9, 0x9D, 0x88, 0x43, + 0xE9, 0x9D, 0x91, 0x43, 0xE9, 0x9D, 0x96, 0x43, + // Bytes 1500 - 153f + 0xE9, 0x9D, 0x9E, 0x43, 0xE9, 0x9D, 0xA2, 0x43, + 0xE9, 0x9D, 0xA9, 0x43, 0xE9, 0x9F, 0x8B, 0x43, + 0xE9, 0x9F, 0x9B, 0x43, 0xE9, 0x9F, 0xA0, 0x43, + 0xE9, 0x9F, 0xAD, 0x43, 0xE9, 0x9F, 0xB3, 0x43, + 0xE9, 0x9F, 0xBF, 0x43, 0xE9, 0xA0, 0x81, 0x43, + 0xE9, 0xA0, 0x85, 0x43, 0xE9, 0xA0, 0x8B, 0x43, + 0xE9, 0xA0, 0x98, 0x43, 0xE9, 0xA0, 0xA9, 0x43, + 0xE9, 0xA0, 0xBB, 0x43, 0xE9, 0xA1, 0x9E, 0x43, + // Bytes 1540 - 157f + 0xE9, 0xA2, 0xA8, 0x43, 0xE9, 0xA3, 0x9B, 0x43, + 0xE9, 0xA3, 0x9F, 0x43, 0xE9, 0xA3, 0xA2, 0x43, + 0xE9, 0xA3, 0xAF, 0x43, 0xE9, 0xA3, 0xBC, 0x43, + 0xE9, 0xA4, 0xA8, 0x43, 0xE9, 0xA4, 0xA9, 0x43, + 0xE9, 0xA6, 0x96, 0x43, 0xE9, 0xA6, 0x99, 0x43, + 0xE9, 0xA6, 0xA7, 0x43, 0xE9, 0xA6, 0xAC, 0x43, + 0xE9, 0xA7, 0x82, 0x43, 0xE9, 0xA7, 0xB1, 0x43, + 0xE9, 0xA7, 0xBE, 0x43, 0xE9, 0xA9, 0xAA, 0x43, + // Bytes 1580 - 15bf + 0xE9, 0xAA, 0xA8, 0x43, 0xE9, 0xAB, 0x98, 0x43, + 0xE9, 0xAB, 0x9F, 0x43, 0xE9, 0xAC, 0x92, 0x43, + 0xE9, 0xAC, 0xA5, 0x43, 0xE9, 0xAC, 0xAF, 0x43, + 0xE9, 0xAC, 0xB2, 0x43, 0xE9, 0xAC, 0xBC, 0x43, + 0xE9, 0xAD, 0x9A, 0x43, 0xE9, 0xAD, 0xAF, 0x43, + 0xE9, 0xB1, 0x80, 0x43, 0xE9, 0xB1, 0x97, 0x43, + 0xE9, 0xB3, 0xA5, 0x43, 0xE9, 0xB3, 0xBD, 0x43, + 0xE9, 0xB5, 0xA7, 0x43, 0xE9, 0xB6, 0xB4, 0x43, + // Bytes 15c0 - 15ff + 0xE9, 0xB7, 0xBA, 0x43, 0xE9, 0xB8, 0x9E, 0x43, + 0xE9, 0xB9, 0xB5, 0x43, 0xE9, 0xB9, 0xBF, 0x43, + 0xE9, 0xBA, 0x97, 0x43, 0xE9, 0xBA, 0x9F, 0x43, + 0xE9, 0xBA, 0xA5, 0x43, 0xE9, 0xBA, 0xBB, 0x43, + 0xE9, 0xBB, 0x83, 0x43, 0xE9, 0xBB, 0x8D, 0x43, + 0xE9, 0xBB, 0x8E, 0x43, 0xE9, 0xBB, 0x91, 0x43, + 0xE9, 0xBB, 0xB9, 0x43, 0xE9, 0xBB, 0xBD, 0x43, + 0xE9, 0xBB, 0xBE, 0x43, 0xE9, 0xBC, 0x85, 0x43, + // Bytes 1600 - 163f + 0xE9, 0xBC, 0x8E, 0x43, 0xE9, 0xBC, 0x8F, 0x43, + 0xE9, 0xBC, 0x93, 0x43, 0xE9, 0xBC, 0x96, 0x43, + 0xE9, 0xBC, 0xA0, 0x43, 0xE9, 0xBC, 0xBB, 0x43, + 0xE9, 0xBD, 0x83, 0x43, 0xE9, 0xBD, 0x8A, 0x43, + 0xE9, 0xBD, 0x92, 0x43, 0xE9, 0xBE, 0x8D, 0x43, + 0xE9, 0xBE, 0x8E, 0x43, 0xE9, 0xBE, 0x9C, 0x43, + 0xE9, 0xBE, 0x9F, 0x43, 0xE9, 0xBE, 0xA0, 0x43, + 0xEA, 0x9C, 0xA7, 0x43, 0xEA, 0x9D, 0xAF, 0x43, + // Bytes 1640 - 167f + 0xEA, 0xAC, 0xB7, 0x43, 0xEA, 0xAD, 0x92, 0x44, + 0xF0, 0xA0, 0x84, 0xA2, 0x44, 0xF0, 0xA0, 0x94, + 0x9C, 0x44, 0xF0, 0xA0, 0x94, 0xA5, 0x44, 0xF0, + 0xA0, 0x95, 0x8B, 0x44, 0xF0, 0xA0, 0x98, 0xBA, + 0x44, 0xF0, 0xA0, 0xA0, 0x84, 0x44, 0xF0, 0xA0, + 0xA3, 0x9E, 0x44, 0xF0, 0xA0, 0xA8, 0xAC, 0x44, + 0xF0, 0xA0, 0xAD, 0xA3, 0x44, 0xF0, 0xA1, 0x93, + 0xA4, 0x44, 0xF0, 0xA1, 0x9A, 0xA8, 0x44, 0xF0, + // Bytes 1680 - 16bf + 0xA1, 0x9B, 0xAA, 0x44, 0xF0, 0xA1, 0xA7, 0x88, + 0x44, 0xF0, 0xA1, 0xAC, 0x98, 0x44, 0xF0, 0xA1, + 0xB4, 0x8B, 0x44, 0xF0, 0xA1, 0xB7, 0xA4, 0x44, + 0xF0, 0xA1, 0xB7, 0xA6, 0x44, 0xF0, 0xA2, 0x86, + 0x83, 0x44, 0xF0, 0xA2, 0x86, 0x9F, 0x44, 0xF0, + 0xA2, 0x8C, 0xB1, 0x44, 0xF0, 0xA2, 0x9B, 0x94, + 0x44, 0xF0, 0xA2, 0xA1, 0x84, 0x44, 0xF0, 0xA2, + 0xA1, 0x8A, 0x44, 0xF0, 0xA2, 0xAC, 0x8C, 0x44, + // Bytes 16c0 - 16ff + 0xF0, 0xA2, 0xAF, 0xB1, 0x44, 0xF0, 0xA3, 0x80, + 0x8A, 0x44, 0xF0, 0xA3, 0x8A, 0xB8, 0x44, 0xF0, + 0xA3, 0x8D, 0x9F, 0x44, 0xF0, 0xA3, 0x8E, 0x93, + 0x44, 0xF0, 0xA3, 0x8E, 0x9C, 0x44, 0xF0, 0xA3, + 0x8F, 0x83, 0x44, 0xF0, 0xA3, 0x8F, 0x95, 0x44, + 0xF0, 0xA3, 0x91, 0xAD, 0x44, 0xF0, 0xA3, 0x9A, + 0xA3, 0x44, 0xF0, 0xA3, 0xA2, 0xA7, 0x44, 0xF0, + 0xA3, 0xAA, 0x8D, 0x44, 0xF0, 0xA3, 0xAB, 0xBA, + // Bytes 1700 - 173f + 0x44, 0xF0, 0xA3, 0xB2, 0xBC, 0x44, 0xF0, 0xA3, + 0xB4, 0x9E, 0x44, 0xF0, 0xA3, 0xBB, 0x91, 0x44, + 0xF0, 0xA3, 0xBD, 0x9E, 0x44, 0xF0, 0xA3, 0xBE, + 0x8E, 0x44, 0xF0, 0xA4, 0x89, 0xA3, 0x44, 0xF0, + 0xA4, 0x8B, 0xAE, 0x44, 0xF0, 0xA4, 0x8E, 0xAB, + 0x44, 0xF0, 0xA4, 0x98, 0x88, 0x44, 0xF0, 0xA4, + 0x9C, 0xB5, 0x44, 0xF0, 0xA4, 0xA0, 0x94, 0x44, + 0xF0, 0xA4, 0xB0, 0xB6, 0x44, 0xF0, 0xA4, 0xB2, + // Bytes 1740 - 177f + 0x92, 0x44, 0xF0, 0xA4, 0xBE, 0xA1, 0x44, 0xF0, + 0xA4, 0xBE, 0xB8, 0x44, 0xF0, 0xA5, 0x81, 0x84, + 0x44, 0xF0, 0xA5, 0x83, 0xB2, 0x44, 0xF0, 0xA5, + 0x83, 0xB3, 0x44, 0xF0, 0xA5, 0x84, 0x99, 0x44, + 0xF0, 0xA5, 0x84, 0xB3, 0x44, 0xF0, 0xA5, 0x89, + 0x89, 0x44, 0xF0, 0xA5, 0x90, 0x9D, 0x44, 0xF0, + 0xA5, 0x98, 0xA6, 0x44, 0xF0, 0xA5, 0x9A, 0x9A, + 0x44, 0xF0, 0xA5, 0x9B, 0x85, 0x44, 0xF0, 0xA5, + // Bytes 1780 - 17bf + 0xA5, 0xBC, 0x44, 0xF0, 0xA5, 0xAA, 0xA7, 0x44, + 0xF0, 0xA5, 0xAE, 0xAB, 0x44, 0xF0, 0xA5, 0xB2, + 0x80, 0x44, 0xF0, 0xA5, 0xB3, 0x90, 0x44, 0xF0, + 0xA5, 0xBE, 0x86, 0x44, 0xF0, 0xA6, 0x87, 0x9A, + 0x44, 0xF0, 0xA6, 0x88, 0xA8, 0x44, 0xF0, 0xA6, + 0x89, 0x87, 0x44, 0xF0, 0xA6, 0x8B, 0x99, 0x44, + 0xF0, 0xA6, 0x8C, 0xBE, 0x44, 0xF0, 0xA6, 0x93, + 0x9A, 0x44, 0xF0, 0xA6, 0x94, 0xA3, 0x44, 0xF0, + // Bytes 17c0 - 17ff + 0xA6, 0x96, 0xA8, 0x44, 0xF0, 0xA6, 0x9E, 0xA7, + 0x44, 0xF0, 0xA6, 0x9E, 0xB5, 0x44, 0xF0, 0xA6, + 0xAC, 0xBC, 0x44, 0xF0, 0xA6, 0xB0, 0xB6, 0x44, + 0xF0, 0xA6, 0xB3, 0x95, 0x44, 0xF0, 0xA6, 0xB5, + 0xAB, 0x44, 0xF0, 0xA6, 0xBC, 0xAC, 0x44, 0xF0, + 0xA6, 0xBE, 0xB1, 0x44, 0xF0, 0xA7, 0x83, 0x92, + 0x44, 0xF0, 0xA7, 0x8F, 0x8A, 0x44, 0xF0, 0xA7, + 0x99, 0xA7, 0x44, 0xF0, 0xA7, 0xA2, 0xAE, 0x44, + // Bytes 1800 - 183f + 0xF0, 0xA7, 0xA5, 0xA6, 0x44, 0xF0, 0xA7, 0xB2, + 0xA8, 0x44, 0xF0, 0xA7, 0xBB, 0x93, 0x44, 0xF0, + 0xA7, 0xBC, 0xAF, 0x44, 0xF0, 0xA8, 0x97, 0x92, + 0x44, 0xF0, 0xA8, 0x97, 0xAD, 0x44, 0xF0, 0xA8, + 0x9C, 0xAE, 0x44, 0xF0, 0xA8, 0xAF, 0xBA, 0x44, + 0xF0, 0xA8, 0xB5, 0xB7, 0x44, 0xF0, 0xA9, 0x85, + 0x85, 0x44, 0xF0, 0xA9, 0x87, 0x9F, 0x44, 0xF0, + 0xA9, 0x88, 0x9A, 0x44, 0xF0, 0xA9, 0x90, 0x8A, + // Bytes 1840 - 187f + 0x44, 0xF0, 0xA9, 0x92, 0x96, 0x44, 0xF0, 0xA9, + 0x96, 0xB6, 0x44, 0xF0, 0xA9, 0xAC, 0xB0, 0x44, + 0xF0, 0xAA, 0x83, 0x8E, 0x44, 0xF0, 0xAA, 0x84, + 0x85, 0x44, 0xF0, 0xAA, 0x88, 0x8E, 0x44, 0xF0, + 0xAA, 0x8A, 0x91, 0x44, 0xF0, 0xAA, 0x8E, 0x92, + 0x44, 0xF0, 0xAA, 0x98, 0x80, 0x42, 0x21, 0x21, + 0x42, 0x21, 0x3F, 0x42, 0x2E, 0x2E, 0x42, 0x30, + 0x2C, 0x42, 0x30, 0x2E, 0x42, 0x31, 0x2C, 0x42, + // Bytes 1880 - 18bf + 0x31, 0x2E, 0x42, 0x31, 0x30, 0x42, 0x31, 0x31, + 0x42, 0x31, 0x32, 0x42, 0x31, 0x33, 0x42, 0x31, + 0x34, 0x42, 0x31, 0x35, 0x42, 0x31, 0x36, 0x42, + 0x31, 0x37, 0x42, 0x31, 0x38, 0x42, 0x31, 0x39, + 0x42, 0x32, 0x2C, 0x42, 0x32, 0x2E, 0x42, 0x32, + 0x30, 0x42, 0x32, 0x31, 0x42, 0x32, 0x32, 0x42, + 0x32, 0x33, 0x42, 0x32, 0x34, 0x42, 0x32, 0x35, + 0x42, 0x32, 0x36, 0x42, 0x32, 0x37, 0x42, 0x32, + // Bytes 18c0 - 18ff + 0x38, 0x42, 0x32, 0x39, 0x42, 0x33, 0x2C, 0x42, + 0x33, 0x2E, 0x42, 0x33, 0x30, 0x42, 0x33, 0x31, + 0x42, 0x33, 0x32, 0x42, 0x33, 0x33, 0x42, 0x33, + 0x34, 0x42, 0x33, 0x35, 0x42, 0x33, 0x36, 0x42, + 0x33, 0x37, 0x42, 0x33, 0x38, 0x42, 0x33, 0x39, + 0x42, 0x34, 0x2C, 0x42, 0x34, 0x2E, 0x42, 0x34, + 0x30, 0x42, 0x34, 0x31, 0x42, 0x34, 0x32, 0x42, + 0x34, 0x33, 0x42, 0x34, 0x34, 0x42, 0x34, 0x35, + // Bytes 1900 - 193f + 0x42, 0x34, 0x36, 0x42, 0x34, 0x37, 0x42, 0x34, + 0x38, 0x42, 0x34, 0x39, 0x42, 0x35, 0x2C, 0x42, + 0x35, 0x2E, 0x42, 0x35, 0x30, 0x42, 0x36, 0x2C, + 0x42, 0x36, 0x2E, 0x42, 0x37, 0x2C, 0x42, 0x37, + 0x2E, 0x42, 0x38, 0x2C, 0x42, 0x38, 0x2E, 0x42, + 0x39, 0x2C, 0x42, 0x39, 0x2E, 0x42, 0x3D, 0x3D, + 0x42, 0x3F, 0x21, 0x42, 0x3F, 0x3F, 0x42, 0x41, + 0x55, 0x42, 0x42, 0x71, 0x42, 0x43, 0x44, 0x42, + // Bytes 1940 - 197f + 0x44, 0x4A, 0x42, 0x44, 0x5A, 0x42, 0x44, 0x7A, + 0x42, 0x47, 0x42, 0x42, 0x47, 0x79, 0x42, 0x48, + 0x50, 0x42, 0x48, 0x56, 0x42, 0x48, 0x67, 0x42, + 0x48, 0x7A, 0x42, 0x49, 0x49, 0x42, 0x49, 0x4A, + 0x42, 0x49, 0x55, 0x42, 0x49, 0x56, 0x42, 0x49, + 0x58, 0x42, 0x4B, 0x42, 0x42, 0x4B, 0x4B, 0x42, + 0x4B, 0x4D, 0x42, 0x4C, 0x4A, 0x42, 0x4C, 0x6A, + 0x42, 0x4D, 0x42, 0x42, 0x4D, 0x43, 0x42, 0x4D, + // Bytes 1980 - 19bf + 0x44, 0x42, 0x4D, 0x56, 0x42, 0x4D, 0x57, 0x42, + 0x4E, 0x4A, 0x42, 0x4E, 0x6A, 0x42, 0x4E, 0x6F, + 0x42, 0x50, 0x48, 0x42, 0x50, 0x52, 0x42, 0x50, + 0x61, 0x42, 0x52, 0x73, 0x42, 0x53, 0x44, 0x42, + 0x53, 0x4D, 0x42, 0x53, 0x53, 0x42, 0x53, 0x76, + 0x42, 0x54, 0x4D, 0x42, 0x56, 0x49, 0x42, 0x57, + 0x43, 0x42, 0x57, 0x5A, 0x42, 0x57, 0x62, 0x42, + 0x58, 0x49, 0x42, 0x63, 0x63, 0x42, 0x63, 0x64, + // Bytes 19c0 - 19ff + 0x42, 0x63, 0x6D, 0x42, 0x64, 0x42, 0x42, 0x64, + 0x61, 0x42, 0x64, 0x6C, 0x42, 0x64, 0x6D, 0x42, + 0x64, 0x7A, 0x42, 0x65, 0x56, 0x42, 0x66, 0x66, + 0x42, 0x66, 0x69, 0x42, 0x66, 0x6C, 0x42, 0x66, + 0x6D, 0x42, 0x68, 0x61, 0x42, 0x69, 0x69, 0x42, + 0x69, 0x6A, 0x42, 0x69, 0x6E, 0x42, 0x69, 0x76, + 0x42, 0x69, 0x78, 0x42, 0x6B, 0x41, 0x42, 0x6B, + 0x56, 0x42, 0x6B, 0x57, 0x42, 0x6B, 0x67, 0x42, + // Bytes 1a00 - 1a3f + 0x6B, 0x6C, 0x42, 0x6B, 0x6D, 0x42, 0x6B, 0x74, + 0x42, 0x6C, 0x6A, 0x42, 0x6C, 0x6D, 0x42, 0x6C, + 0x6E, 0x42, 0x6C, 0x78, 0x42, 0x6D, 0x32, 0x42, + 0x6D, 0x33, 0x42, 0x6D, 0x41, 0x42, 0x6D, 0x56, + 0x42, 0x6D, 0x57, 0x42, 0x6D, 0x62, 0x42, 0x6D, + 0x67, 0x42, 0x6D, 0x6C, 0x42, 0x6D, 0x6D, 0x42, + 0x6D, 0x73, 0x42, 0x6E, 0x41, 0x42, 0x6E, 0x46, + 0x42, 0x6E, 0x56, 0x42, 0x6E, 0x57, 0x42, 0x6E, + // Bytes 1a40 - 1a7f + 0x6A, 0x42, 0x6E, 0x6D, 0x42, 0x6E, 0x73, 0x42, + 0x6F, 0x56, 0x42, 0x70, 0x41, 0x42, 0x70, 0x46, + 0x42, 0x70, 0x56, 0x42, 0x70, 0x57, 0x42, 0x70, + 0x63, 0x42, 0x70, 0x73, 0x42, 0x73, 0x72, 0x42, + 0x73, 0x74, 0x42, 0x76, 0x69, 0x42, 0x78, 0x69, + 0x43, 0x28, 0x31, 0x29, 0x43, 0x28, 0x32, 0x29, + 0x43, 0x28, 0x33, 0x29, 0x43, 0x28, 0x34, 0x29, + 0x43, 0x28, 0x35, 0x29, 0x43, 0x28, 0x36, 0x29, + // Bytes 1a80 - 1abf + 0x43, 0x28, 0x37, 0x29, 0x43, 0x28, 0x38, 0x29, + 0x43, 0x28, 0x39, 0x29, 0x43, 0x28, 0x41, 0x29, + 0x43, 0x28, 0x42, 0x29, 0x43, 0x28, 0x43, 0x29, + 0x43, 0x28, 0x44, 0x29, 0x43, 0x28, 0x45, 0x29, + 0x43, 0x28, 0x46, 0x29, 0x43, 0x28, 0x47, 0x29, + 0x43, 0x28, 0x48, 0x29, 0x43, 0x28, 0x49, 0x29, + 0x43, 0x28, 0x4A, 0x29, 0x43, 0x28, 0x4B, 0x29, + 0x43, 0x28, 0x4C, 0x29, 0x43, 0x28, 0x4D, 0x29, + // Bytes 1ac0 - 1aff + 0x43, 0x28, 0x4E, 0x29, 0x43, 0x28, 0x4F, 0x29, + 0x43, 0x28, 0x50, 0x29, 0x43, 0x28, 0x51, 0x29, + 0x43, 0x28, 0x52, 0x29, 0x43, 0x28, 0x53, 0x29, + 0x43, 0x28, 0x54, 0x29, 0x43, 0x28, 0x55, 0x29, + 0x43, 0x28, 0x56, 0x29, 0x43, 0x28, 0x57, 0x29, + 0x43, 0x28, 0x58, 0x29, 0x43, 0x28, 0x59, 0x29, + 0x43, 0x28, 0x5A, 0x29, 0x43, 0x28, 0x61, 0x29, + 0x43, 0x28, 0x62, 0x29, 0x43, 0x28, 0x63, 0x29, + // Bytes 1b00 - 1b3f + 0x43, 0x28, 0x64, 0x29, 0x43, 0x28, 0x65, 0x29, + 0x43, 0x28, 0x66, 0x29, 0x43, 0x28, 0x67, 0x29, + 0x43, 0x28, 0x68, 0x29, 0x43, 0x28, 0x69, 0x29, + 0x43, 0x28, 0x6A, 0x29, 0x43, 0x28, 0x6B, 0x29, + 0x43, 0x28, 0x6C, 0x29, 0x43, 0x28, 0x6D, 0x29, + 0x43, 0x28, 0x6E, 0x29, 0x43, 0x28, 0x6F, 0x29, + 0x43, 0x28, 0x70, 0x29, 0x43, 0x28, 0x71, 0x29, + 0x43, 0x28, 0x72, 0x29, 0x43, 0x28, 0x73, 0x29, + // Bytes 1b40 - 1b7f + 0x43, 0x28, 0x74, 0x29, 0x43, 0x28, 0x75, 0x29, + 0x43, 0x28, 0x76, 0x29, 0x43, 0x28, 0x77, 0x29, + 0x43, 0x28, 0x78, 0x29, 0x43, 0x28, 0x79, 0x29, + 0x43, 0x28, 0x7A, 0x29, 0x43, 0x2E, 0x2E, 0x2E, + 0x43, 0x31, 0x30, 0x2E, 0x43, 0x31, 0x31, 0x2E, + 0x43, 0x31, 0x32, 0x2E, 0x43, 0x31, 0x33, 0x2E, + 0x43, 0x31, 0x34, 0x2E, 0x43, 0x31, 0x35, 0x2E, + 0x43, 0x31, 0x36, 0x2E, 0x43, 0x31, 0x37, 0x2E, + // Bytes 1b80 - 1bbf + 0x43, 0x31, 0x38, 0x2E, 0x43, 0x31, 0x39, 0x2E, + 0x43, 0x32, 0x30, 0x2E, 0x43, 0x3A, 0x3A, 0x3D, + 0x43, 0x3D, 0x3D, 0x3D, 0x43, 0x43, 0x6F, 0x2E, + 0x43, 0x46, 0x41, 0x58, 0x43, 0x47, 0x48, 0x7A, + 0x43, 0x47, 0x50, 0x61, 0x43, 0x49, 0x49, 0x49, + 0x43, 0x4C, 0x54, 0x44, 0x43, 0x4C, 0xC2, 0xB7, + 0x43, 0x4D, 0x48, 0x7A, 0x43, 0x4D, 0x50, 0x61, + 0x43, 0x4D, 0xCE, 0xA9, 0x43, 0x50, 0x50, 0x4D, + // Bytes 1bc0 - 1bff + 0x43, 0x50, 0x50, 0x56, 0x43, 0x50, 0x54, 0x45, + 0x43, 0x54, 0x45, 0x4C, 0x43, 0x54, 0x48, 0x7A, + 0x43, 0x56, 0x49, 0x49, 0x43, 0x58, 0x49, 0x49, + 0x43, 0x61, 0x2F, 0x63, 0x43, 0x61, 0x2F, 0x73, + 0x43, 0x61, 0xCA, 0xBE, 0x43, 0x62, 0x61, 0x72, + 0x43, 0x63, 0x2F, 0x6F, 0x43, 0x63, 0x2F, 0x75, + 0x43, 0x63, 0x61, 0x6C, 0x43, 0x63, 0x6D, 0x32, + 0x43, 0x63, 0x6D, 0x33, 0x43, 0x64, 0x6D, 0x32, + // Bytes 1c00 - 1c3f + 0x43, 0x64, 0x6D, 0x33, 0x43, 0x65, 0x72, 0x67, + 0x43, 0x66, 0x66, 0x69, 0x43, 0x66, 0x66, 0x6C, + 0x43, 0x67, 0x61, 0x6C, 0x43, 0x68, 0x50, 0x61, + 0x43, 0x69, 0x69, 0x69, 0x43, 0x6B, 0x48, 0x7A, + 0x43, 0x6B, 0x50, 0x61, 0x43, 0x6B, 0x6D, 0x32, + 0x43, 0x6B, 0x6D, 0x33, 0x43, 0x6B, 0xCE, 0xA9, + 0x43, 0x6C, 0x6F, 0x67, 0x43, 0x6C, 0xC2, 0xB7, + 0x43, 0x6D, 0x69, 0x6C, 0x43, 0x6D, 0x6D, 0x32, + // Bytes 1c40 - 1c7f + 0x43, 0x6D, 0x6D, 0x33, 0x43, 0x6D, 0x6F, 0x6C, + 0x43, 0x72, 0x61, 0x64, 0x43, 0x76, 0x69, 0x69, + 0x43, 0x78, 0x69, 0x69, 0x43, 0xC2, 0xB0, 0x43, + 0x43, 0xC2, 0xB0, 0x46, 0x43, 0xCA, 0xBC, 0x6E, + 0x43, 0xCE, 0xBC, 0x41, 0x43, 0xCE, 0xBC, 0x46, + 0x43, 0xCE, 0xBC, 0x56, 0x43, 0xCE, 0xBC, 0x57, + 0x43, 0xCE, 0xBC, 0x67, 0x43, 0xCE, 0xBC, 0x6C, + 0x43, 0xCE, 0xBC, 0x6D, 0x43, 0xCE, 0xBC, 0x73, + // Bytes 1c80 - 1cbf + 0x44, 0x28, 0x31, 0x30, 0x29, 0x44, 0x28, 0x31, + 0x31, 0x29, 0x44, 0x28, 0x31, 0x32, 0x29, 0x44, + 0x28, 0x31, 0x33, 0x29, 0x44, 0x28, 0x31, 0x34, + 0x29, 0x44, 0x28, 0x31, 0x35, 0x29, 0x44, 0x28, + 0x31, 0x36, 0x29, 0x44, 0x28, 0x31, 0x37, 0x29, + 0x44, 0x28, 0x31, 0x38, 0x29, 0x44, 0x28, 0x31, + 0x39, 0x29, 0x44, 0x28, 0x32, 0x30, 0x29, 0x44, + 0x30, 0xE7, 0x82, 0xB9, 0x44, 0x31, 0xE2, 0x81, + // Bytes 1cc0 - 1cff + 0x84, 0x44, 0x31, 0xE6, 0x97, 0xA5, 0x44, 0x31, + 0xE6, 0x9C, 0x88, 0x44, 0x31, 0xE7, 0x82, 0xB9, + 0x44, 0x32, 0xE6, 0x97, 0xA5, 0x44, 0x32, 0xE6, + 0x9C, 0x88, 0x44, 0x32, 0xE7, 0x82, 0xB9, 0x44, + 0x33, 0xE6, 0x97, 0xA5, 0x44, 0x33, 0xE6, 0x9C, + 0x88, 0x44, 0x33, 0xE7, 0x82, 0xB9, 0x44, 0x34, + 0xE6, 0x97, 0xA5, 0x44, 0x34, 0xE6, 0x9C, 0x88, + 0x44, 0x34, 0xE7, 0x82, 0xB9, 0x44, 0x35, 0xE6, + // Bytes 1d00 - 1d3f + 0x97, 0xA5, 0x44, 0x35, 0xE6, 0x9C, 0x88, 0x44, + 0x35, 0xE7, 0x82, 0xB9, 0x44, 0x36, 0xE6, 0x97, + 0xA5, 0x44, 0x36, 0xE6, 0x9C, 0x88, 0x44, 0x36, + 0xE7, 0x82, 0xB9, 0x44, 0x37, 0xE6, 0x97, 0xA5, + 0x44, 0x37, 0xE6, 0x9C, 0x88, 0x44, 0x37, 0xE7, + 0x82, 0xB9, 0x44, 0x38, 0xE6, 0x97, 0xA5, 0x44, + 0x38, 0xE6, 0x9C, 0x88, 0x44, 0x38, 0xE7, 0x82, + 0xB9, 0x44, 0x39, 0xE6, 0x97, 0xA5, 0x44, 0x39, + // Bytes 1d40 - 1d7f + 0xE6, 0x9C, 0x88, 0x44, 0x39, 0xE7, 0x82, 0xB9, + 0x44, 0x56, 0x49, 0x49, 0x49, 0x44, 0x61, 0x2E, + 0x6D, 0x2E, 0x44, 0x6B, 0x63, 0x61, 0x6C, 0x44, + 0x70, 0x2E, 0x6D, 0x2E, 0x44, 0x76, 0x69, 0x69, + 0x69, 0x44, 0xD5, 0xA5, 0xD6, 0x82, 0x44, 0xD5, + 0xB4, 0xD5, 0xA5, 0x44, 0xD5, 0xB4, 0xD5, 0xAB, + 0x44, 0xD5, 0xB4, 0xD5, 0xAD, 0x44, 0xD5, 0xB4, + 0xD5, 0xB6, 0x44, 0xD5, 0xBE, 0xD5, 0xB6, 0x44, + // Bytes 1d80 - 1dbf + 0xD7, 0x90, 0xD7, 0x9C, 0x44, 0xD8, 0xA7, 0xD9, + 0xB4, 0x44, 0xD8, 0xA8, 0xD8, 0xAC, 0x44, 0xD8, + 0xA8, 0xD8, 0xAD, 0x44, 0xD8, 0xA8, 0xD8, 0xAE, + 0x44, 0xD8, 0xA8, 0xD8, 0xB1, 0x44, 0xD8, 0xA8, + 0xD8, 0xB2, 0x44, 0xD8, 0xA8, 0xD9, 0x85, 0x44, + 0xD8, 0xA8, 0xD9, 0x86, 0x44, 0xD8, 0xA8, 0xD9, + 0x87, 0x44, 0xD8, 0xA8, 0xD9, 0x89, 0x44, 0xD8, + 0xA8, 0xD9, 0x8A, 0x44, 0xD8, 0xAA, 0xD8, 0xAC, + // Bytes 1dc0 - 1dff + 0x44, 0xD8, 0xAA, 0xD8, 0xAD, 0x44, 0xD8, 0xAA, + 0xD8, 0xAE, 0x44, 0xD8, 0xAA, 0xD8, 0xB1, 0x44, + 0xD8, 0xAA, 0xD8, 0xB2, 0x44, 0xD8, 0xAA, 0xD9, + 0x85, 0x44, 0xD8, 0xAA, 0xD9, 0x86, 0x44, 0xD8, + 0xAA, 0xD9, 0x87, 0x44, 0xD8, 0xAA, 0xD9, 0x89, + 0x44, 0xD8, 0xAA, 0xD9, 0x8A, 0x44, 0xD8, 0xAB, + 0xD8, 0xAC, 0x44, 0xD8, 0xAB, 0xD8, 0xB1, 0x44, + 0xD8, 0xAB, 0xD8, 0xB2, 0x44, 0xD8, 0xAB, 0xD9, + // Bytes 1e00 - 1e3f + 0x85, 0x44, 0xD8, 0xAB, 0xD9, 0x86, 0x44, 0xD8, + 0xAB, 0xD9, 0x87, 0x44, 0xD8, 0xAB, 0xD9, 0x89, + 0x44, 0xD8, 0xAB, 0xD9, 0x8A, 0x44, 0xD8, 0xAC, + 0xD8, 0xAD, 0x44, 0xD8, 0xAC, 0xD9, 0x85, 0x44, + 0xD8, 0xAC, 0xD9, 0x89, 0x44, 0xD8, 0xAC, 0xD9, + 0x8A, 0x44, 0xD8, 0xAD, 0xD8, 0xAC, 0x44, 0xD8, + 0xAD, 0xD9, 0x85, 0x44, 0xD8, 0xAD, 0xD9, 0x89, + 0x44, 0xD8, 0xAD, 0xD9, 0x8A, 0x44, 0xD8, 0xAE, + // Bytes 1e40 - 1e7f + 0xD8, 0xAC, 0x44, 0xD8, 0xAE, 0xD8, 0xAD, 0x44, + 0xD8, 0xAE, 0xD9, 0x85, 0x44, 0xD8, 0xAE, 0xD9, + 0x89, 0x44, 0xD8, 0xAE, 0xD9, 0x8A, 0x44, 0xD8, + 0xB3, 0xD8, 0xAC, 0x44, 0xD8, 0xB3, 0xD8, 0xAD, + 0x44, 0xD8, 0xB3, 0xD8, 0xAE, 0x44, 0xD8, 0xB3, + 0xD8, 0xB1, 0x44, 0xD8, 0xB3, 0xD9, 0x85, 0x44, + 0xD8, 0xB3, 0xD9, 0x87, 0x44, 0xD8, 0xB3, 0xD9, + 0x89, 0x44, 0xD8, 0xB3, 0xD9, 0x8A, 0x44, 0xD8, + // Bytes 1e80 - 1ebf + 0xB4, 0xD8, 0xAC, 0x44, 0xD8, 0xB4, 0xD8, 0xAD, + 0x44, 0xD8, 0xB4, 0xD8, 0xAE, 0x44, 0xD8, 0xB4, + 0xD8, 0xB1, 0x44, 0xD8, 0xB4, 0xD9, 0x85, 0x44, + 0xD8, 0xB4, 0xD9, 0x87, 0x44, 0xD8, 0xB4, 0xD9, + 0x89, 0x44, 0xD8, 0xB4, 0xD9, 0x8A, 0x44, 0xD8, + 0xB5, 0xD8, 0xAD, 0x44, 0xD8, 0xB5, 0xD8, 0xAE, + 0x44, 0xD8, 0xB5, 0xD8, 0xB1, 0x44, 0xD8, 0xB5, + 0xD9, 0x85, 0x44, 0xD8, 0xB5, 0xD9, 0x89, 0x44, + // Bytes 1ec0 - 1eff + 0xD8, 0xB5, 0xD9, 0x8A, 0x44, 0xD8, 0xB6, 0xD8, + 0xAC, 0x44, 0xD8, 0xB6, 0xD8, 0xAD, 0x44, 0xD8, + 0xB6, 0xD8, 0xAE, 0x44, 0xD8, 0xB6, 0xD8, 0xB1, + 0x44, 0xD8, 0xB6, 0xD9, 0x85, 0x44, 0xD8, 0xB6, + 0xD9, 0x89, 0x44, 0xD8, 0xB6, 0xD9, 0x8A, 0x44, + 0xD8, 0xB7, 0xD8, 0xAD, 0x44, 0xD8, 0xB7, 0xD9, + 0x85, 0x44, 0xD8, 0xB7, 0xD9, 0x89, 0x44, 0xD8, + 0xB7, 0xD9, 0x8A, 0x44, 0xD8, 0xB8, 0xD9, 0x85, + // Bytes 1f00 - 1f3f + 0x44, 0xD8, 0xB9, 0xD8, 0xAC, 0x44, 0xD8, 0xB9, + 0xD9, 0x85, 0x44, 0xD8, 0xB9, 0xD9, 0x89, 0x44, + 0xD8, 0xB9, 0xD9, 0x8A, 0x44, 0xD8, 0xBA, 0xD8, + 0xAC, 0x44, 0xD8, 0xBA, 0xD9, 0x85, 0x44, 0xD8, + 0xBA, 0xD9, 0x89, 0x44, 0xD8, 0xBA, 0xD9, 0x8A, + 0x44, 0xD9, 0x81, 0xD8, 0xAC, 0x44, 0xD9, 0x81, + 0xD8, 0xAD, 0x44, 0xD9, 0x81, 0xD8, 0xAE, 0x44, + 0xD9, 0x81, 0xD9, 0x85, 0x44, 0xD9, 0x81, 0xD9, + // Bytes 1f40 - 1f7f + 0x89, 0x44, 0xD9, 0x81, 0xD9, 0x8A, 0x44, 0xD9, + 0x82, 0xD8, 0xAD, 0x44, 0xD9, 0x82, 0xD9, 0x85, + 0x44, 0xD9, 0x82, 0xD9, 0x89, 0x44, 0xD9, 0x82, + 0xD9, 0x8A, 0x44, 0xD9, 0x83, 0xD8, 0xA7, 0x44, + 0xD9, 0x83, 0xD8, 0xAC, 0x44, 0xD9, 0x83, 0xD8, + 0xAD, 0x44, 0xD9, 0x83, 0xD8, 0xAE, 0x44, 0xD9, + 0x83, 0xD9, 0x84, 0x44, 0xD9, 0x83, 0xD9, 0x85, + 0x44, 0xD9, 0x83, 0xD9, 0x89, 0x44, 0xD9, 0x83, + // Bytes 1f80 - 1fbf + 0xD9, 0x8A, 0x44, 0xD9, 0x84, 0xD8, 0xA7, 0x44, + 0xD9, 0x84, 0xD8, 0xAC, 0x44, 0xD9, 0x84, 0xD8, + 0xAD, 0x44, 0xD9, 0x84, 0xD8, 0xAE, 0x44, 0xD9, + 0x84, 0xD9, 0x85, 0x44, 0xD9, 0x84, 0xD9, 0x87, + 0x44, 0xD9, 0x84, 0xD9, 0x89, 0x44, 0xD9, 0x84, + 0xD9, 0x8A, 0x44, 0xD9, 0x85, 0xD8, 0xA7, 0x44, + 0xD9, 0x85, 0xD8, 0xAC, 0x44, 0xD9, 0x85, 0xD8, + 0xAD, 0x44, 0xD9, 0x85, 0xD8, 0xAE, 0x44, 0xD9, + // Bytes 1fc0 - 1fff + 0x85, 0xD9, 0x85, 0x44, 0xD9, 0x85, 0xD9, 0x89, + 0x44, 0xD9, 0x85, 0xD9, 0x8A, 0x44, 0xD9, 0x86, + 0xD8, 0xAC, 0x44, 0xD9, 0x86, 0xD8, 0xAD, 0x44, + 0xD9, 0x86, 0xD8, 0xAE, 0x44, 0xD9, 0x86, 0xD8, + 0xB1, 0x44, 0xD9, 0x86, 0xD8, 0xB2, 0x44, 0xD9, + 0x86, 0xD9, 0x85, 0x44, 0xD9, 0x86, 0xD9, 0x86, + 0x44, 0xD9, 0x86, 0xD9, 0x87, 0x44, 0xD9, 0x86, + 0xD9, 0x89, 0x44, 0xD9, 0x86, 0xD9, 0x8A, 0x44, + // Bytes 2000 - 203f + 0xD9, 0x87, 0xD8, 0xAC, 0x44, 0xD9, 0x87, 0xD9, + 0x85, 0x44, 0xD9, 0x87, 0xD9, 0x89, 0x44, 0xD9, + 0x87, 0xD9, 0x8A, 0x44, 0xD9, 0x88, 0xD9, 0xB4, + 0x44, 0xD9, 0x8A, 0xD8, 0xAC, 0x44, 0xD9, 0x8A, + 0xD8, 0xAD, 0x44, 0xD9, 0x8A, 0xD8, 0xAE, 0x44, + 0xD9, 0x8A, 0xD8, 0xB1, 0x44, 0xD9, 0x8A, 0xD8, + 0xB2, 0x44, 0xD9, 0x8A, 0xD9, 0x85, 0x44, 0xD9, + 0x8A, 0xD9, 0x86, 0x44, 0xD9, 0x8A, 0xD9, 0x87, + // Bytes 2040 - 207f + 0x44, 0xD9, 0x8A, 0xD9, 0x89, 0x44, 0xD9, 0x8A, + 0xD9, 0x8A, 0x44, 0xD9, 0x8A, 0xD9, 0xB4, 0x44, + 0xDB, 0x87, 0xD9, 0xB4, 0x45, 0x28, 0xE1, 0x84, + 0x80, 0x29, 0x45, 0x28, 0xE1, 0x84, 0x82, 0x29, + 0x45, 0x28, 0xE1, 0x84, 0x83, 0x29, 0x45, 0x28, + 0xE1, 0x84, 0x85, 0x29, 0x45, 0x28, 0xE1, 0x84, + 0x86, 0x29, 0x45, 0x28, 0xE1, 0x84, 0x87, 0x29, + 0x45, 0x28, 0xE1, 0x84, 0x89, 0x29, 0x45, 0x28, + // Bytes 2080 - 20bf + 0xE1, 0x84, 0x8B, 0x29, 0x45, 0x28, 0xE1, 0x84, + 0x8C, 0x29, 0x45, 0x28, 0xE1, 0x84, 0x8E, 0x29, + 0x45, 0x28, 0xE1, 0x84, 0x8F, 0x29, 0x45, 0x28, + 0xE1, 0x84, 0x90, 0x29, 0x45, 0x28, 0xE1, 0x84, + 0x91, 0x29, 0x45, 0x28, 0xE1, 0x84, 0x92, 0x29, + 0x45, 0x28, 0xE4, 0xB8, 0x80, 0x29, 0x45, 0x28, + 0xE4, 0xB8, 0x83, 0x29, 0x45, 0x28, 0xE4, 0xB8, + 0x89, 0x29, 0x45, 0x28, 0xE4, 0xB9, 0x9D, 0x29, + // Bytes 20c0 - 20ff + 0x45, 0x28, 0xE4, 0xBA, 0x8C, 0x29, 0x45, 0x28, + 0xE4, 0xBA, 0x94, 0x29, 0x45, 0x28, 0xE4, 0xBB, + 0xA3, 0x29, 0x45, 0x28, 0xE4, 0xBC, 0x81, 0x29, + 0x45, 0x28, 0xE4, 0xBC, 0x91, 0x29, 0x45, 0x28, + 0xE5, 0x85, 0xAB, 0x29, 0x45, 0x28, 0xE5, 0x85, + 0xAD, 0x29, 0x45, 0x28, 0xE5, 0x8A, 0xB4, 0x29, + 0x45, 0x28, 0xE5, 0x8D, 0x81, 0x29, 0x45, 0x28, + 0xE5, 0x8D, 0x94, 0x29, 0x45, 0x28, 0xE5, 0x90, + // Bytes 2100 - 213f + 0x8D, 0x29, 0x45, 0x28, 0xE5, 0x91, 0xBC, 0x29, + 0x45, 0x28, 0xE5, 0x9B, 0x9B, 0x29, 0x45, 0x28, + 0xE5, 0x9C, 0x9F, 0x29, 0x45, 0x28, 0xE5, 0xAD, + 0xA6, 0x29, 0x45, 0x28, 0xE6, 0x97, 0xA5, 0x29, + 0x45, 0x28, 0xE6, 0x9C, 0x88, 0x29, 0x45, 0x28, + 0xE6, 0x9C, 0x89, 0x29, 0x45, 0x28, 0xE6, 0x9C, + 0xA8, 0x29, 0x45, 0x28, 0xE6, 0xA0, 0xAA, 0x29, + 0x45, 0x28, 0xE6, 0xB0, 0xB4, 0x29, 0x45, 0x28, + // Bytes 2140 - 217f + 0xE7, 0x81, 0xAB, 0x29, 0x45, 0x28, 0xE7, 0x89, + 0xB9, 0x29, 0x45, 0x28, 0xE7, 0x9B, 0xA3, 0x29, + 0x45, 0x28, 0xE7, 0xA4, 0xBE, 0x29, 0x45, 0x28, + 0xE7, 0xA5, 0x9D, 0x29, 0x45, 0x28, 0xE7, 0xA5, + 0xAD, 0x29, 0x45, 0x28, 0xE8, 0x87, 0xAA, 0x29, + 0x45, 0x28, 0xE8, 0x87, 0xB3, 0x29, 0x45, 0x28, + 0xE8, 0xB2, 0xA1, 0x29, 0x45, 0x28, 0xE8, 0xB3, + 0x87, 0x29, 0x45, 0x28, 0xE9, 0x87, 0x91, 0x29, + // Bytes 2180 - 21bf + 0x45, 0x30, 0xE2, 0x81, 0x84, 0x33, 0x45, 0x31, + 0x30, 0xE6, 0x97, 0xA5, 0x45, 0x31, 0x30, 0xE6, + 0x9C, 0x88, 0x45, 0x31, 0x30, 0xE7, 0x82, 0xB9, + 0x45, 0x31, 0x31, 0xE6, 0x97, 0xA5, 0x45, 0x31, + 0x31, 0xE6, 0x9C, 0x88, 0x45, 0x31, 0x31, 0xE7, + 0x82, 0xB9, 0x45, 0x31, 0x32, 0xE6, 0x97, 0xA5, + 0x45, 0x31, 0x32, 0xE6, 0x9C, 0x88, 0x45, 0x31, + 0x32, 0xE7, 0x82, 0xB9, 0x45, 0x31, 0x33, 0xE6, + // Bytes 21c0 - 21ff + 0x97, 0xA5, 0x45, 0x31, 0x33, 0xE7, 0x82, 0xB9, + 0x45, 0x31, 0x34, 0xE6, 0x97, 0xA5, 0x45, 0x31, + 0x34, 0xE7, 0x82, 0xB9, 0x45, 0x31, 0x35, 0xE6, + 0x97, 0xA5, 0x45, 0x31, 0x35, 0xE7, 0x82, 0xB9, + 0x45, 0x31, 0x36, 0xE6, 0x97, 0xA5, 0x45, 0x31, + 0x36, 0xE7, 0x82, 0xB9, 0x45, 0x31, 0x37, 0xE6, + 0x97, 0xA5, 0x45, 0x31, 0x37, 0xE7, 0x82, 0xB9, + 0x45, 0x31, 0x38, 0xE6, 0x97, 0xA5, 0x45, 0x31, + // Bytes 2200 - 223f + 0x38, 0xE7, 0x82, 0xB9, 0x45, 0x31, 0x39, 0xE6, + 0x97, 0xA5, 0x45, 0x31, 0x39, 0xE7, 0x82, 0xB9, + 0x45, 0x31, 0xE2, 0x81, 0x84, 0x32, 0x45, 0x31, + 0xE2, 0x81, 0x84, 0x33, 0x45, 0x31, 0xE2, 0x81, + 0x84, 0x34, 0x45, 0x31, 0xE2, 0x81, 0x84, 0x35, + 0x45, 0x31, 0xE2, 0x81, 0x84, 0x36, 0x45, 0x31, + 0xE2, 0x81, 0x84, 0x37, 0x45, 0x31, 0xE2, 0x81, + 0x84, 0x38, 0x45, 0x31, 0xE2, 0x81, 0x84, 0x39, + // Bytes 2240 - 227f + 0x45, 0x32, 0x30, 0xE6, 0x97, 0xA5, 0x45, 0x32, + 0x30, 0xE7, 0x82, 0xB9, 0x45, 0x32, 0x31, 0xE6, + 0x97, 0xA5, 0x45, 0x32, 0x31, 0xE7, 0x82, 0xB9, + 0x45, 0x32, 0x32, 0xE6, 0x97, 0xA5, 0x45, 0x32, + 0x32, 0xE7, 0x82, 0xB9, 0x45, 0x32, 0x33, 0xE6, + 0x97, 0xA5, 0x45, 0x32, 0x33, 0xE7, 0x82, 0xB9, + 0x45, 0x32, 0x34, 0xE6, 0x97, 0xA5, 0x45, 0x32, + 0x34, 0xE7, 0x82, 0xB9, 0x45, 0x32, 0x35, 0xE6, + // Bytes 2280 - 22bf + 0x97, 0xA5, 0x45, 0x32, 0x36, 0xE6, 0x97, 0xA5, + 0x45, 0x32, 0x37, 0xE6, 0x97, 0xA5, 0x45, 0x32, + 0x38, 0xE6, 0x97, 0xA5, 0x45, 0x32, 0x39, 0xE6, + 0x97, 0xA5, 0x45, 0x32, 0xE2, 0x81, 0x84, 0x33, + 0x45, 0x32, 0xE2, 0x81, 0x84, 0x35, 0x45, 0x33, + 0x30, 0xE6, 0x97, 0xA5, 0x45, 0x33, 0x31, 0xE6, + 0x97, 0xA5, 0x45, 0x33, 0xE2, 0x81, 0x84, 0x34, + 0x45, 0x33, 0xE2, 0x81, 0x84, 0x35, 0x45, 0x33, + // Bytes 22c0 - 22ff + 0xE2, 0x81, 0x84, 0x38, 0x45, 0x34, 0xE2, 0x81, + 0x84, 0x35, 0x45, 0x35, 0xE2, 0x81, 0x84, 0x36, + 0x45, 0x35, 0xE2, 0x81, 0x84, 0x38, 0x45, 0x37, + 0xE2, 0x81, 0x84, 0x38, 0x45, 0x41, 0xE2, 0x88, + 0x95, 0x6D, 0x45, 0x56, 0xE2, 0x88, 0x95, 0x6D, + 0x45, 0x6D, 0xE2, 0x88, 0x95, 0x73, 0x46, 0x31, + 0xE2, 0x81, 0x84, 0x31, 0x30, 0x46, 0x43, 0xE2, + 0x88, 0x95, 0x6B, 0x67, 0x46, 0x6D, 0xE2, 0x88, + // Bytes 2300 - 233f + 0x95, 0x73, 0x32, 0x46, 0xD8, 0xA8, 0xD8, 0xAD, + 0xD9, 0x8A, 0x46, 0xD8, 0xA8, 0xD8, 0xAE, 0xD9, + 0x8A, 0x46, 0xD8, 0xAA, 0xD8, 0xAC, 0xD9, 0x85, + 0x46, 0xD8, 0xAA, 0xD8, 0xAC, 0xD9, 0x89, 0x46, + 0xD8, 0xAA, 0xD8, 0xAC, 0xD9, 0x8A, 0x46, 0xD8, + 0xAA, 0xD8, 0xAD, 0xD8, 0xAC, 0x46, 0xD8, 0xAA, + 0xD8, 0xAD, 0xD9, 0x85, 0x46, 0xD8, 0xAA, 0xD8, + 0xAE, 0xD9, 0x85, 0x46, 0xD8, 0xAA, 0xD8, 0xAE, + // Bytes 2340 - 237f + 0xD9, 0x89, 0x46, 0xD8, 0xAA, 0xD8, 0xAE, 0xD9, + 0x8A, 0x46, 0xD8, 0xAA, 0xD9, 0x85, 0xD8, 0xAC, + 0x46, 0xD8, 0xAA, 0xD9, 0x85, 0xD8, 0xAD, 0x46, + 0xD8, 0xAA, 0xD9, 0x85, 0xD8, 0xAE, 0x46, 0xD8, + 0xAA, 0xD9, 0x85, 0xD9, 0x89, 0x46, 0xD8, 0xAA, + 0xD9, 0x85, 0xD9, 0x8A, 0x46, 0xD8, 0xAC, 0xD8, + 0xAD, 0xD9, 0x89, 0x46, 0xD8, 0xAC, 0xD8, 0xAD, + 0xD9, 0x8A, 0x46, 0xD8, 0xAC, 0xD9, 0x85, 0xD8, + // Bytes 2380 - 23bf + 0xAD, 0x46, 0xD8, 0xAC, 0xD9, 0x85, 0xD9, 0x89, + 0x46, 0xD8, 0xAC, 0xD9, 0x85, 0xD9, 0x8A, 0x46, + 0xD8, 0xAD, 0xD8, 0xAC, 0xD9, 0x8A, 0x46, 0xD8, + 0xAD, 0xD9, 0x85, 0xD9, 0x89, 0x46, 0xD8, 0xAD, + 0xD9, 0x85, 0xD9, 0x8A, 0x46, 0xD8, 0xB3, 0xD8, + 0xAC, 0xD8, 0xAD, 0x46, 0xD8, 0xB3, 0xD8, 0xAC, + 0xD9, 0x89, 0x46, 0xD8, 0xB3, 0xD8, 0xAD, 0xD8, + 0xAC, 0x46, 0xD8, 0xB3, 0xD8, 0xAE, 0xD9, 0x89, + // Bytes 23c0 - 23ff + 0x46, 0xD8, 0xB3, 0xD8, 0xAE, 0xD9, 0x8A, 0x46, + 0xD8, 0xB3, 0xD9, 0x85, 0xD8, 0xAC, 0x46, 0xD8, + 0xB3, 0xD9, 0x85, 0xD8, 0xAD, 0x46, 0xD8, 0xB3, + 0xD9, 0x85, 0xD9, 0x85, 0x46, 0xD8, 0xB4, 0xD8, + 0xAC, 0xD9, 0x8A, 0x46, 0xD8, 0xB4, 0xD8, 0xAD, + 0xD9, 0x85, 0x46, 0xD8, 0xB4, 0xD8, 0xAD, 0xD9, + 0x8A, 0x46, 0xD8, 0xB4, 0xD9, 0x85, 0xD8, 0xAE, + 0x46, 0xD8, 0xB4, 0xD9, 0x85, 0xD9, 0x85, 0x46, + // Bytes 2400 - 243f + 0xD8, 0xB5, 0xD8, 0xAD, 0xD8, 0xAD, 0x46, 0xD8, + 0xB5, 0xD8, 0xAD, 0xD9, 0x8A, 0x46, 0xD8, 0xB5, + 0xD9, 0x84, 0xD9, 0x89, 0x46, 0xD8, 0xB5, 0xD9, + 0x84, 0xDB, 0x92, 0x46, 0xD8, 0xB5, 0xD9, 0x85, + 0xD9, 0x85, 0x46, 0xD8, 0xB6, 0xD8, 0xAD, 0xD9, + 0x89, 0x46, 0xD8, 0xB6, 0xD8, 0xAD, 0xD9, 0x8A, + 0x46, 0xD8, 0xB6, 0xD8, 0xAE, 0xD9, 0x85, 0x46, + 0xD8, 0xB7, 0xD9, 0x85, 0xD8, 0xAD, 0x46, 0xD8, + // Bytes 2440 - 247f + 0xB7, 0xD9, 0x85, 0xD9, 0x85, 0x46, 0xD8, 0xB7, + 0xD9, 0x85, 0xD9, 0x8A, 0x46, 0xD8, 0xB9, 0xD8, + 0xAC, 0xD9, 0x85, 0x46, 0xD8, 0xB9, 0xD9, 0x85, + 0xD9, 0x85, 0x46, 0xD8, 0xB9, 0xD9, 0x85, 0xD9, + 0x89, 0x46, 0xD8, 0xB9, 0xD9, 0x85, 0xD9, 0x8A, + 0x46, 0xD8, 0xBA, 0xD9, 0x85, 0xD9, 0x85, 0x46, + 0xD8, 0xBA, 0xD9, 0x85, 0xD9, 0x89, 0x46, 0xD8, + 0xBA, 0xD9, 0x85, 0xD9, 0x8A, 0x46, 0xD9, 0x81, + // Bytes 2480 - 24bf + 0xD8, 0xAE, 0xD9, 0x85, 0x46, 0xD9, 0x81, 0xD9, + 0x85, 0xD9, 0x8A, 0x46, 0xD9, 0x82, 0xD9, 0x84, + 0xDB, 0x92, 0x46, 0xD9, 0x82, 0xD9, 0x85, 0xD8, + 0xAD, 0x46, 0xD9, 0x82, 0xD9, 0x85, 0xD9, 0x85, + 0x46, 0xD9, 0x82, 0xD9, 0x85, 0xD9, 0x8A, 0x46, + 0xD9, 0x83, 0xD9, 0x85, 0xD9, 0x85, 0x46, 0xD9, + 0x83, 0xD9, 0x85, 0xD9, 0x8A, 0x46, 0xD9, 0x84, + 0xD8, 0xAC, 0xD8, 0xAC, 0x46, 0xD9, 0x84, 0xD8, + // Bytes 24c0 - 24ff + 0xAC, 0xD9, 0x85, 0x46, 0xD9, 0x84, 0xD8, 0xAC, + 0xD9, 0x8A, 0x46, 0xD9, 0x84, 0xD8, 0xAD, 0xD9, + 0x85, 0x46, 0xD9, 0x84, 0xD8, 0xAD, 0xD9, 0x89, + 0x46, 0xD9, 0x84, 0xD8, 0xAD, 0xD9, 0x8A, 0x46, + 0xD9, 0x84, 0xD8, 0xAE, 0xD9, 0x85, 0x46, 0xD9, + 0x84, 0xD9, 0x85, 0xD8, 0xAD, 0x46, 0xD9, 0x84, + 0xD9, 0x85, 0xD9, 0x8A, 0x46, 0xD9, 0x85, 0xD8, + 0xAC, 0xD8, 0xAD, 0x46, 0xD9, 0x85, 0xD8, 0xAC, + // Bytes 2500 - 253f + 0xD8, 0xAE, 0x46, 0xD9, 0x85, 0xD8, 0xAC, 0xD9, + 0x85, 0x46, 0xD9, 0x85, 0xD8, 0xAC, 0xD9, 0x8A, + 0x46, 0xD9, 0x85, 0xD8, 0xAD, 0xD8, 0xAC, 0x46, + 0xD9, 0x85, 0xD8, 0xAD, 0xD9, 0x85, 0x46, 0xD9, + 0x85, 0xD8, 0xAD, 0xD9, 0x8A, 0x46, 0xD9, 0x85, + 0xD8, 0xAE, 0xD8, 0xAC, 0x46, 0xD9, 0x85, 0xD8, + 0xAE, 0xD9, 0x85, 0x46, 0xD9, 0x85, 0xD8, 0xAE, + 0xD9, 0x8A, 0x46, 0xD9, 0x85, 0xD9, 0x85, 0xD9, + // Bytes 2540 - 257f + 0x8A, 0x46, 0xD9, 0x86, 0xD8, 0xAC, 0xD8, 0xAD, + 0x46, 0xD9, 0x86, 0xD8, 0xAC, 0xD9, 0x85, 0x46, + 0xD9, 0x86, 0xD8, 0xAC, 0xD9, 0x89, 0x46, 0xD9, + 0x86, 0xD8, 0xAC, 0xD9, 0x8A, 0x46, 0xD9, 0x86, + 0xD8, 0xAD, 0xD9, 0x85, 0x46, 0xD9, 0x86, 0xD8, + 0xAD, 0xD9, 0x89, 0x46, 0xD9, 0x86, 0xD8, 0xAD, + 0xD9, 0x8A, 0x46, 0xD9, 0x86, 0xD9, 0x85, 0xD9, + 0x89, 0x46, 0xD9, 0x86, 0xD9, 0x85, 0xD9, 0x8A, + // Bytes 2580 - 25bf + 0x46, 0xD9, 0x87, 0xD9, 0x85, 0xD8, 0xAC, 0x46, + 0xD9, 0x87, 0xD9, 0x85, 0xD9, 0x85, 0x46, 0xD9, + 0x8A, 0xD8, 0xAC, 0xD9, 0x8A, 0x46, 0xD9, 0x8A, + 0xD8, 0xAD, 0xD9, 0x8A, 0x46, 0xD9, 0x8A, 0xD9, + 0x85, 0xD9, 0x85, 0x46, 0xD9, 0x8A, 0xD9, 0x85, + 0xD9, 0x8A, 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xD8, + 0xA7, 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xD8, 0xAC, + 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xD8, 0xAD, 0x46, + // Bytes 25c0 - 25ff + 0xD9, 0x8A, 0xD9, 0x94, 0xD8, 0xAE, 0x46, 0xD9, + 0x8A, 0xD9, 0x94, 0xD8, 0xB1, 0x46, 0xD9, 0x8A, + 0xD9, 0x94, 0xD8, 0xB2, 0x46, 0xD9, 0x8A, 0xD9, + 0x94, 0xD9, 0x85, 0x46, 0xD9, 0x8A, 0xD9, 0x94, + 0xD9, 0x86, 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xD9, + 0x87, 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xD9, 0x88, + 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xD9, 0x89, 0x46, + 0xD9, 0x8A, 0xD9, 0x94, 0xD9, 0x8A, 0x46, 0xD9, + // Bytes 2600 - 263f + 0x8A, 0xD9, 0x94, 0xDB, 0x86, 0x46, 0xD9, 0x8A, + 0xD9, 0x94, 0xDB, 0x87, 0x46, 0xD9, 0x8A, 0xD9, + 0x94, 0xDB, 0x88, 0x46, 0xD9, 0x8A, 0xD9, 0x94, + 0xDB, 0x90, 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xDB, + 0x95, 0x46, 0xE0, 0xB9, 0x8D, 0xE0, 0xB8, 0xB2, + 0x46, 0xE0, 0xBA, 0xAB, 0xE0, 0xBA, 0x99, 0x46, + 0xE0, 0xBA, 0xAB, 0xE0, 0xBA, 0xA1, 0x46, 0xE0, + 0xBB, 0x8D, 0xE0, 0xBA, 0xB2, 0x46, 0xE0, 0xBD, + // Bytes 2640 - 267f + 0x80, 0xE0, 0xBE, 0xB5, 0x46, 0xE0, 0xBD, 0x82, + 0xE0, 0xBE, 0xB7, 0x46, 0xE0, 0xBD, 0x8C, 0xE0, + 0xBE, 0xB7, 0x46, 0xE0, 0xBD, 0x91, 0xE0, 0xBE, + 0xB7, 0x46, 0xE0, 0xBD, 0x96, 0xE0, 0xBE, 0xB7, + 0x46, 0xE0, 0xBD, 0x9B, 0xE0, 0xBE, 0xB7, 0x46, + 0xE0, 0xBE, 0x90, 0xE0, 0xBE, 0xB5, 0x46, 0xE0, + 0xBE, 0x92, 0xE0, 0xBE, 0xB7, 0x46, 0xE0, 0xBE, + 0x9C, 0xE0, 0xBE, 0xB7, 0x46, 0xE0, 0xBE, 0xA1, + // Bytes 2680 - 26bf + 0xE0, 0xBE, 0xB7, 0x46, 0xE0, 0xBE, 0xA6, 0xE0, + 0xBE, 0xB7, 0x46, 0xE0, 0xBE, 0xAB, 0xE0, 0xBE, + 0xB7, 0x46, 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, + 0x46, 0xE2, 0x80, 0xB5, 0xE2, 0x80, 0xB5, 0x46, + 0xE2, 0x88, 0xAB, 0xE2, 0x88, 0xAB, 0x46, 0xE2, + 0x88, 0xAE, 0xE2, 0x88, 0xAE, 0x46, 0xE3, 0x81, + 0xBB, 0xE3, 0x81, 0x8B, 0x46, 0xE3, 0x82, 0x88, + 0xE3, 0x82, 0x8A, 0x46, 0xE3, 0x82, 0xAD, 0xE3, + // Bytes 26c0 - 26ff + 0x83, 0xAD, 0x46, 0xE3, 0x82, 0xB3, 0xE3, 0x82, + 0xB3, 0x46, 0xE3, 0x82, 0xB3, 0xE3, 0x83, 0x88, + 0x46, 0xE3, 0x83, 0x88, 0xE3, 0x83, 0xB3, 0x46, + 0xE3, 0x83, 0x8A, 0xE3, 0x83, 0x8E, 0x46, 0xE3, + 0x83, 0x9B, 0xE3, 0x83, 0xB3, 0x46, 0xE3, 0x83, + 0x9F, 0xE3, 0x83, 0xAA, 0x46, 0xE3, 0x83, 0xAA, + 0xE3, 0x83, 0xA9, 0x46, 0xE3, 0x83, 0xAC, 0xE3, + 0x83, 0xA0, 0x46, 0xE5, 0xA4, 0xA7, 0xE6, 0xAD, + // Bytes 2700 - 273f + 0xA3, 0x46, 0xE5, 0xB9, 0xB3, 0xE6, 0x88, 0x90, + 0x46, 0xE6, 0x98, 0x8E, 0xE6, 0xB2, 0xBB, 0x46, + 0xE6, 0x98, 0xAD, 0xE5, 0x92, 0x8C, 0x47, 0x72, + 0x61, 0x64, 0xE2, 0x88, 0x95, 0x73, 0x47, 0xE3, + 0x80, 0x94, 0x53, 0xE3, 0x80, 0x95, 0x48, 0x28, + 0xE1, 0x84, 0x80, 0xE1, 0x85, 0xA1, 0x29, 0x48, + 0x28, 0xE1, 0x84, 0x82, 0xE1, 0x85, 0xA1, 0x29, + 0x48, 0x28, 0xE1, 0x84, 0x83, 0xE1, 0x85, 0xA1, + // Bytes 2740 - 277f + 0x29, 0x48, 0x28, 0xE1, 0x84, 0x85, 0xE1, 0x85, + 0xA1, 0x29, 0x48, 0x28, 0xE1, 0x84, 0x86, 0xE1, + 0x85, 0xA1, 0x29, 0x48, 0x28, 0xE1, 0x84, 0x87, + 0xE1, 0x85, 0xA1, 0x29, 0x48, 0x28, 0xE1, 0x84, + 0x89, 0xE1, 0x85, 0xA1, 0x29, 0x48, 0x28, 0xE1, + 0x84, 0x8B, 0xE1, 0x85, 0xA1, 0x29, 0x48, 0x28, + 0xE1, 0x84, 0x8C, 0xE1, 0x85, 0xA1, 0x29, 0x48, + 0x28, 0xE1, 0x84, 0x8C, 0xE1, 0x85, 0xAE, 0x29, + // Bytes 2780 - 27bf + 0x48, 0x28, 0xE1, 0x84, 0x8E, 0xE1, 0x85, 0xA1, + 0x29, 0x48, 0x28, 0xE1, 0x84, 0x8F, 0xE1, 0x85, + 0xA1, 0x29, 0x48, 0x28, 0xE1, 0x84, 0x90, 0xE1, + 0x85, 0xA1, 0x29, 0x48, 0x28, 0xE1, 0x84, 0x91, + 0xE1, 0x85, 0xA1, 0x29, 0x48, 0x28, 0xE1, 0x84, + 0x92, 0xE1, 0x85, 0xA1, 0x29, 0x48, 0x72, 0x61, + 0x64, 0xE2, 0x88, 0x95, 0x73, 0x32, 0x48, 0xD8, + 0xA7, 0xD9, 0x83, 0xD8, 0xA8, 0xD8, 0xB1, 0x48, + // Bytes 27c0 - 27ff + 0xD8, 0xA7, 0xD9, 0x84, 0xD9, 0x84, 0xD9, 0x87, + 0x48, 0xD8, 0xB1, 0xD8, 0xB3, 0xD9, 0x88, 0xD9, + 0x84, 0x48, 0xD8, 0xB1, 0xDB, 0x8C, 0xD8, 0xA7, + 0xD9, 0x84, 0x48, 0xD8, 0xB5, 0xD9, 0x84, 0xD8, + 0xB9, 0xD9, 0x85, 0x48, 0xD8, 0xB9, 0xD9, 0x84, + 0xD9, 0x8A, 0xD9, 0x87, 0x48, 0xD9, 0x85, 0xD8, + 0xAD, 0xD9, 0x85, 0xD8, 0xAF, 0x48, 0xD9, 0x88, + 0xD8, 0xB3, 0xD9, 0x84, 0xD9, 0x85, 0x49, 0xE2, + // Bytes 2800 - 283f + 0x80, 0xB2, 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, + 0x49, 0xE2, 0x80, 0xB5, 0xE2, 0x80, 0xB5, 0xE2, + 0x80, 0xB5, 0x49, 0xE2, 0x88, 0xAB, 0xE2, 0x88, + 0xAB, 0xE2, 0x88, 0xAB, 0x49, 0xE2, 0x88, 0xAE, + 0xE2, 0x88, 0xAE, 0xE2, 0x88, 0xAE, 0x49, 0xE3, + 0x80, 0x94, 0xE4, 0xB8, 0x89, 0xE3, 0x80, 0x95, + 0x49, 0xE3, 0x80, 0x94, 0xE4, 0xBA, 0x8C, 0xE3, + 0x80, 0x95, 0x49, 0xE3, 0x80, 0x94, 0xE5, 0x8B, + // Bytes 2840 - 287f + 0x9D, 0xE3, 0x80, 0x95, 0x49, 0xE3, 0x80, 0x94, + 0xE5, 0xAE, 0x89, 0xE3, 0x80, 0x95, 0x49, 0xE3, + 0x80, 0x94, 0xE6, 0x89, 0x93, 0xE3, 0x80, 0x95, + 0x49, 0xE3, 0x80, 0x94, 0xE6, 0x95, 0x97, 0xE3, + 0x80, 0x95, 0x49, 0xE3, 0x80, 0x94, 0xE6, 0x9C, + 0xAC, 0xE3, 0x80, 0x95, 0x49, 0xE3, 0x80, 0x94, + 0xE7, 0x82, 0xB9, 0xE3, 0x80, 0x95, 0x49, 0xE3, + 0x80, 0x94, 0xE7, 0x9B, 0x97, 0xE3, 0x80, 0x95, + // Bytes 2880 - 28bf + 0x49, 0xE3, 0x82, 0xA2, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0xAB, 0x49, 0xE3, 0x82, 0xA4, 0xE3, 0x83, + 0xB3, 0xE3, 0x83, 0x81, 0x49, 0xE3, 0x82, 0xA6, + 0xE3, 0x82, 0xA9, 0xE3, 0x83, 0xB3, 0x49, 0xE3, + 0x82, 0xAA, 0xE3, 0x83, 0xB3, 0xE3, 0x82, 0xB9, + 0x49, 0xE3, 0x82, 0xAA, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0xA0, 0x49, 0xE3, 0x82, 0xAB, 0xE3, 0x82, + 0xA4, 0xE3, 0x83, 0xAA, 0x49, 0xE3, 0x82, 0xB1, + // Bytes 28c0 - 28ff + 0xE3, 0x83, 0xBC, 0xE3, 0x82, 0xB9, 0x49, 0xE3, + 0x82, 0xB3, 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0x8A, + 0x49, 0xE3, 0x82, 0xBB, 0xE3, 0x83, 0xB3, 0xE3, + 0x83, 0x81, 0x49, 0xE3, 0x82, 0xBB, 0xE3, 0x83, + 0xB3, 0xE3, 0x83, 0x88, 0x49, 0xE3, 0x83, 0x86, + 0xE3, 0x82, 0x99, 0xE3, 0x82, 0xB7, 0x49, 0xE3, + 0x83, 0x88, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xAB, + 0x49, 0xE3, 0x83, 0x8E, 0xE3, 0x83, 0x83, 0xE3, + // Bytes 2900 - 293f + 0x83, 0x88, 0x49, 0xE3, 0x83, 0x8F, 0xE3, 0x82, + 0xA4, 0xE3, 0x83, 0x84, 0x49, 0xE3, 0x83, 0x92, + 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xAB, 0x49, 0xE3, + 0x83, 0x92, 0xE3, 0x82, 0x9A, 0xE3, 0x82, 0xB3, + 0x49, 0xE3, 0x83, 0x95, 0xE3, 0x83, 0xA9, 0xE3, + 0x83, 0xB3, 0x49, 0xE3, 0x83, 0x98, 0xE3, 0x82, + 0x9A, 0xE3, 0x82, 0xBD, 0x49, 0xE3, 0x83, 0x98, + 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0x84, 0x49, 0xE3, + // Bytes 2940 - 297f + 0x83, 0x9B, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0xAB, + 0x49, 0xE3, 0x83, 0x9B, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0xB3, 0x49, 0xE3, 0x83, 0x9E, 0xE3, 0x82, + 0xA4, 0xE3, 0x83, 0xAB, 0x49, 0xE3, 0x83, 0x9E, + 0xE3, 0x83, 0x83, 0xE3, 0x83, 0x8F, 0x49, 0xE3, + 0x83, 0x9E, 0xE3, 0x83, 0xAB, 0xE3, 0x82, 0xAF, + 0x49, 0xE3, 0x83, 0xA4, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0xAB, 0x49, 0xE3, 0x83, 0xA6, 0xE3, 0x82, + // Bytes 2980 - 29bf + 0xA2, 0xE3, 0x83, 0xB3, 0x49, 0xE3, 0x83, 0xAF, + 0xE3, 0x83, 0x83, 0xE3, 0x83, 0x88, 0x4C, 0xE2, + 0x80, 0xB2, 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, + 0xE2, 0x80, 0xB2, 0x4C, 0xE2, 0x88, 0xAB, 0xE2, + 0x88, 0xAB, 0xE2, 0x88, 0xAB, 0xE2, 0x88, 0xAB, + 0x4C, 0xE3, 0x82, 0xA2, 0xE3, 0x83, 0xAB, 0xE3, + 0x83, 0x95, 0xE3, 0x82, 0xA1, 0x4C, 0xE3, 0x82, + 0xA8, 0xE3, 0x83, 0xBC, 0xE3, 0x82, 0xAB, 0xE3, + // Bytes 29c0 - 29ff + 0x83, 0xBC, 0x4C, 0xE3, 0x82, 0xAB, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xAD, 0xE3, 0x83, 0xB3, 0x4C, + 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0xE3, 0x83, + 0xB3, 0xE3, 0x83, 0x9E, 0x4C, 0xE3, 0x82, 0xAB, + 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0x83, 0xE3, 0x83, + 0x88, 0x4C, 0xE3, 0x82, 0xAB, 0xE3, 0x83, 0xAD, + 0xE3, 0x83, 0xAA, 0xE3, 0x83, 0xBC, 0x4C, 0xE3, + 0x82, 0xAD, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0x8B, + // Bytes 2a00 - 2a3f + 0xE3, 0x83, 0xBC, 0x4C, 0xE3, 0x82, 0xAD, 0xE3, + 0x83, 0xA5, 0xE3, 0x83, 0xAA, 0xE3, 0x83, 0xBC, + 0x4C, 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, 0xE3, + 0x83, 0xA9, 0xE3, 0x83, 0xA0, 0x4C, 0xE3, 0x82, + 0xAF, 0xE3, 0x83, 0xAD, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0x8D, 0x4C, 0xE3, 0x82, 0xB5, 0xE3, 0x82, + 0xA4, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAB, 0x4C, + 0xE3, 0x82, 0xBF, 0xE3, 0x82, 0x99, 0xE3, 0x83, + // Bytes 2a40 - 2a7f + 0xBC, 0xE3, 0x82, 0xB9, 0x4C, 0xE3, 0x83, 0x8F, + 0xE3, 0x82, 0x9A, 0xE3, 0x83, 0xBC, 0xE3, 0x83, + 0x84, 0x4C, 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x9A, + 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAB, 0x4C, 0xE3, + 0x83, 0x95, 0xE3, 0x82, 0xA3, 0xE3, 0x83, 0xBC, + 0xE3, 0x83, 0x88, 0x4C, 0xE3, 0x83, 0x98, 0xE3, + 0x82, 0x99, 0xE3, 0x83, 0xBC, 0xE3, 0x82, 0xBF, + 0x4C, 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, 0xE3, + // Bytes 2a80 - 2abf + 0x83, 0x8B, 0xE3, 0x83, 0x92, 0x4C, 0xE3, 0x83, + 0x98, 0xE3, 0x82, 0x9A, 0xE3, 0x83, 0xB3, 0xE3, + 0x82, 0xB9, 0x4C, 0xE3, 0x83, 0x9B, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0x88, 0x4C, + 0xE3, 0x83, 0x9E, 0xE3, 0x82, 0xA4, 0xE3, 0x82, + 0xAF, 0xE3, 0x83, 0xAD, 0x4C, 0xE3, 0x83, 0x9F, + 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAD, 0xE3, 0x83, + 0xB3, 0x4C, 0xE3, 0x83, 0xA1, 0xE3, 0x83, 0xBC, + // Bytes 2ac0 - 2aff + 0xE3, 0x83, 0x88, 0xE3, 0x83, 0xAB, 0x4C, 0xE3, + 0x83, 0xAA, 0xE3, 0x83, 0x83, 0xE3, 0x83, 0x88, + 0xE3, 0x83, 0xAB, 0x4C, 0xE3, 0x83, 0xAB, 0xE3, + 0x83, 0x92, 0xE3, 0x82, 0x9A, 0xE3, 0x83, 0xBC, + 0x4C, 0xE6, 0xA0, 0xAA, 0xE5, 0xBC, 0x8F, 0xE4, + 0xBC, 0x9A, 0xE7, 0xA4, 0xBE, 0x4E, 0x28, 0xE1, + 0x84, 0x8B, 0xE1, 0x85, 0xA9, 0xE1, 0x84, 0x92, + 0xE1, 0x85, 0xAE, 0x29, 0x4F, 0xD8, 0xAC, 0xD9, + // Bytes 2b00 - 2b3f + 0x84, 0x20, 0xD8, 0xAC, 0xD9, 0x84, 0xD8, 0xA7, + 0xD9, 0x84, 0xD9, 0x87, 0x4F, 0xE3, 0x82, 0xA2, + 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x9A, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0x88, 0x4F, 0xE3, 0x82, 0xA2, + 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x98, 0xE3, 0x82, + 0x9A, 0xE3, 0x82, 0xA2, 0x4F, 0xE3, 0x82, 0xAD, + 0xE3, 0x83, 0xAD, 0xE3, 0x83, 0xAF, 0xE3, 0x83, + 0x83, 0xE3, 0x83, 0x88, 0x4F, 0xE3, 0x82, 0xB5, + // Bytes 2b40 - 2b7f + 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x81, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0xA0, 0x4F, 0xE3, 0x83, 0x8F, + 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xBC, 0xE3, 0x83, + 0xAC, 0xE3, 0x83, 0xAB, 0x4F, 0xE3, 0x83, 0x98, + 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0xBF, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0xAB, 0x4F, 0xE3, 0x83, 0x9B, + 0xE3, 0x82, 0x9A, 0xE3, 0x82, 0xA4, 0xE3, 0x83, + 0xB3, 0xE3, 0x83, 0x88, 0x4F, 0xE3, 0x83, 0x9E, + // Bytes 2b80 - 2bbf + 0xE3, 0x83, 0xB3, 0xE3, 0x82, 0xB7, 0xE3, 0x83, + 0xA7, 0xE3, 0x83, 0xB3, 0x4F, 0xE3, 0x83, 0xA1, + 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0xE3, 0x83, + 0x88, 0xE3, 0x83, 0xB3, 0x4F, 0xE3, 0x83, 0xAB, + 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x95, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xAB, 0x51, 0x28, 0xE1, 0x84, + 0x8B, 0xE1, 0x85, 0xA9, 0xE1, 0x84, 0x8C, 0xE1, + 0x85, 0xA5, 0xE1, 0x86, 0xAB, 0x29, 0x52, 0xE3, + // Bytes 2bc0 - 2bff + 0x82, 0xAD, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xAB, + 0xE3, 0x82, 0xBF, 0xE3, 0x82, 0x99, 0xE3, 0x83, + 0xBC, 0x52, 0xE3, 0x82, 0xAD, 0xE3, 0x83, 0xAD, + 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, 0xE3, 0x83, + 0xA9, 0xE3, 0x83, 0xA0, 0x52, 0xE3, 0x82, 0xAD, + 0xE3, 0x83, 0xAD, 0xE3, 0x83, 0xA1, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0x88, 0xE3, 0x83, 0xAB, 0x52, + 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, 0xE3, 0x83, + // Bytes 2c00 - 2c3f + 0xA9, 0xE3, 0x83, 0xA0, 0xE3, 0x83, 0x88, 0xE3, + 0x83, 0xB3, 0x52, 0xE3, 0x82, 0xAF, 0xE3, 0x83, + 0xAB, 0xE3, 0x82, 0xBB, 0xE3, 0x82, 0x99, 0xE3, + 0x82, 0xA4, 0xE3, 0x83, 0xAD, 0x52, 0xE3, 0x83, + 0x8F, 0xE3, 0x82, 0x9A, 0xE3, 0x83, 0xBC, 0xE3, + 0x82, 0xBB, 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x88, + 0x52, 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x9A, 0xE3, + 0x82, 0xA2, 0xE3, 0x82, 0xB9, 0xE3, 0x83, 0x88, + // Bytes 2c40 - 2c7f + 0xE3, 0x83, 0xAB, 0x52, 0xE3, 0x83, 0x95, 0xE3, + 0x82, 0x99, 0xE3, 0x83, 0x83, 0xE3, 0x82, 0xB7, + 0xE3, 0x82, 0xA7, 0xE3, 0x83, 0xAB, 0x52, 0xE3, + 0x83, 0x9F, 0xE3, 0x83, 0xAA, 0xE3, 0x83, 0x8F, + 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xBC, 0xE3, 0x83, + 0xAB, 0x52, 0xE3, 0x83, 0xAC, 0xE3, 0x83, 0xB3, + 0xE3, 0x83, 0x88, 0xE3, 0x82, 0xB1, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xB3, 0x61, 0xD8, 0xB5, 0xD9, + // Bytes 2c80 - 2cbf + 0x84, 0xD9, 0x89, 0x20, 0xD8, 0xA7, 0xD9, 0x84, + 0xD9, 0x84, 0xD9, 0x87, 0x20, 0xD8, 0xB9, 0xD9, + 0x84, 0xD9, 0x8A, 0xD9, 0x87, 0x20, 0xD9, 0x88, + 0xD8, 0xB3, 0xD9, 0x84, 0xD9, 0x85, 0x06, 0xE0, + 0xA7, 0x87, 0xE0, 0xA6, 0xBE, 0x01, 0x06, 0xE0, + 0xA7, 0x87, 0xE0, 0xA7, 0x97, 0x01, 0x06, 0xE0, + 0xAD, 0x87, 0xE0, 0xAC, 0xBE, 0x01, 0x06, 0xE0, + 0xAD, 0x87, 0xE0, 0xAD, 0x96, 0x01, 0x06, 0xE0, + // Bytes 2cc0 - 2cff + 0xAD, 0x87, 0xE0, 0xAD, 0x97, 0x01, 0x06, 0xE0, + 0xAE, 0x92, 0xE0, 0xAF, 0x97, 0x01, 0x06, 0xE0, + 0xAF, 0x86, 0xE0, 0xAE, 0xBE, 0x01, 0x06, 0xE0, + 0xAF, 0x86, 0xE0, 0xAF, 0x97, 0x01, 0x06, 0xE0, + 0xAF, 0x87, 0xE0, 0xAE, 0xBE, 0x01, 0x06, 0xE0, + 0xB2, 0xBF, 0xE0, 0xB3, 0x95, 0x01, 0x06, 0xE0, + 0xB3, 0x86, 0xE0, 0xB3, 0x95, 0x01, 0x06, 0xE0, + 0xB3, 0x86, 0xE0, 0xB3, 0x96, 0x01, 0x06, 0xE0, + // Bytes 2d00 - 2d3f + 0xB5, 0x86, 0xE0, 0xB4, 0xBE, 0x01, 0x06, 0xE0, + 0xB5, 0x86, 0xE0, 0xB5, 0x97, 0x01, 0x06, 0xE0, + 0xB5, 0x87, 0xE0, 0xB4, 0xBE, 0x01, 0x06, 0xE0, + 0xB7, 0x99, 0xE0, 0xB7, 0x9F, 0x01, 0x06, 0xE1, + 0x80, 0xA5, 0xE1, 0x80, 0xAE, 0x01, 0x06, 0xE1, + 0xAC, 0x85, 0xE1, 0xAC, 0xB5, 0x01, 0x06, 0xE1, + 0xAC, 0x87, 0xE1, 0xAC, 0xB5, 0x01, 0x06, 0xE1, + 0xAC, 0x89, 0xE1, 0xAC, 0xB5, 0x01, 0x06, 0xE1, + // Bytes 2d40 - 2d7f + 0xAC, 0x8B, 0xE1, 0xAC, 0xB5, 0x01, 0x06, 0xE1, + 0xAC, 0x8D, 0xE1, 0xAC, 0xB5, 0x01, 0x06, 0xE1, + 0xAC, 0x91, 0xE1, 0xAC, 0xB5, 0x01, 0x06, 0xE1, + 0xAC, 0xBA, 0xE1, 0xAC, 0xB5, 0x01, 0x06, 0xE1, + 0xAC, 0xBC, 0xE1, 0xAC, 0xB5, 0x01, 0x06, 0xE1, + 0xAC, 0xBE, 0xE1, 0xAC, 0xB5, 0x01, 0x06, 0xE1, + 0xAC, 0xBF, 0xE1, 0xAC, 0xB5, 0x01, 0x06, 0xE1, + 0xAD, 0x82, 0xE1, 0xAC, 0xB5, 0x01, 0x08, 0xF0, + // Bytes 2d80 - 2dbf + 0x91, 0x84, 0xB1, 0xF0, 0x91, 0x84, 0xA7, 0x01, + 0x08, 0xF0, 0x91, 0x84, 0xB2, 0xF0, 0x91, 0x84, + 0xA7, 0x01, 0x08, 0xF0, 0x91, 0x8D, 0x87, 0xF0, + 0x91, 0x8C, 0xBE, 0x01, 0x08, 0xF0, 0x91, 0x8D, + 0x87, 0xF0, 0x91, 0x8D, 0x97, 0x01, 0x08, 0xF0, + 0x91, 0x92, 0xB9, 0xF0, 0x91, 0x92, 0xB0, 0x01, + 0x08, 0xF0, 0x91, 0x92, 0xB9, 0xF0, 0x91, 0x92, + 0xBA, 0x01, 0x08, 0xF0, 0x91, 0x92, 0xB9, 0xF0, + // Bytes 2dc0 - 2dff + 0x91, 0x92, 0xBD, 0x01, 0x08, 0xF0, 0x91, 0x96, + 0xB8, 0xF0, 0x91, 0x96, 0xAF, 0x01, 0x08, 0xF0, + 0x91, 0x96, 0xB9, 0xF0, 0x91, 0x96, 0xAF, 0x01, + 0x09, 0xE0, 0xB3, 0x86, 0xE0, 0xB3, 0x82, 0xE0, + 0xB3, 0x95, 0x02, 0x09, 0xE0, 0xB7, 0x99, 0xE0, + 0xB7, 0x8F, 0xE0, 0xB7, 0x8A, 0x12, 0x44, 0x44, + 0x5A, 0xCC, 0x8C, 0xC9, 0x44, 0x44, 0x7A, 0xCC, + 0x8C, 0xC9, 0x44, 0x64, 0x7A, 0xCC, 0x8C, 0xC9, + // Bytes 2e00 - 2e3f + 0x46, 0xD9, 0x84, 0xD8, 0xA7, 0xD9, 0x93, 0xC9, + 0x46, 0xD9, 0x84, 0xD8, 0xA7, 0xD9, 0x94, 0xC9, + 0x46, 0xD9, 0x84, 0xD8, 0xA7, 0xD9, 0x95, 0xB5, + 0x46, 0xE1, 0x84, 0x80, 0xE1, 0x85, 0xA1, 0x01, + 0x46, 0xE1, 0x84, 0x82, 0xE1, 0x85, 0xA1, 0x01, + 0x46, 0xE1, 0x84, 0x83, 0xE1, 0x85, 0xA1, 0x01, + 0x46, 0xE1, 0x84, 0x85, 0xE1, 0x85, 0xA1, 0x01, + 0x46, 0xE1, 0x84, 0x86, 0xE1, 0x85, 0xA1, 0x01, + // Bytes 2e40 - 2e7f + 0x46, 0xE1, 0x84, 0x87, 0xE1, 0x85, 0xA1, 0x01, + 0x46, 0xE1, 0x84, 0x89, 0xE1, 0x85, 0xA1, 0x01, + 0x46, 0xE1, 0x84, 0x8B, 0xE1, 0x85, 0xA1, 0x01, + 0x46, 0xE1, 0x84, 0x8B, 0xE1, 0x85, 0xAE, 0x01, + 0x46, 0xE1, 0x84, 0x8C, 0xE1, 0x85, 0xA1, 0x01, + 0x46, 0xE1, 0x84, 0x8E, 0xE1, 0x85, 0xA1, 0x01, + 0x46, 0xE1, 0x84, 0x8F, 0xE1, 0x85, 0xA1, 0x01, + 0x46, 0xE1, 0x84, 0x90, 0xE1, 0x85, 0xA1, 0x01, + // Bytes 2e80 - 2ebf + 0x46, 0xE1, 0x84, 0x91, 0xE1, 0x85, 0xA1, 0x01, + 0x46, 0xE1, 0x84, 0x92, 0xE1, 0x85, 0xA1, 0x01, + 0x49, 0xE3, 0x83, 0xA1, 0xE3, 0x82, 0xAB, 0xE3, + 0x82, 0x99, 0x0D, 0x4C, 0xE1, 0x84, 0x8C, 0xE1, + 0x85, 0xAE, 0xE1, 0x84, 0x8B, 0xE1, 0x85, 0xB4, + 0x01, 0x4C, 0xE3, 0x82, 0xAD, 0xE3, 0x82, 0x99, + 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0x0D, 0x4C, + 0xE3, 0x82, 0xB3, 0xE3, 0x83, 0xBC, 0xE3, 0x83, + // Bytes 2ec0 - 2eff + 0x9B, 0xE3, 0x82, 0x9A, 0x0D, 0x4C, 0xE3, 0x83, + 0xA4, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, 0xE3, + 0x82, 0x99, 0x0D, 0x4F, 0xE1, 0x84, 0x8E, 0xE1, + 0x85, 0xA1, 0xE1, 0x86, 0xB7, 0xE1, 0x84, 0x80, + 0xE1, 0x85, 0xA9, 0x01, 0x4F, 0xE3, 0x82, 0xA4, + 0xE3, 0x83, 0x8B, 0xE3, 0x83, 0xB3, 0xE3, 0x82, + 0xAF, 0xE3, 0x82, 0x99, 0x0D, 0x4F, 0xE3, 0x82, + 0xB7, 0xE3, 0x83, 0xAA, 0xE3, 0x83, 0xB3, 0xE3, + // Bytes 2f00 - 2f3f + 0x82, 0xAF, 0xE3, 0x82, 0x99, 0x0D, 0x4F, 0xE3, + 0x83, 0x98, 0xE3, 0x82, 0x9A, 0xE3, 0x83, 0xBC, + 0xE3, 0x82, 0xB7, 0xE3, 0x82, 0x99, 0x0D, 0x4F, + 0xE3, 0x83, 0x9B, 0xE3, 0x82, 0x9A, 0xE3, 0x83, + 0xB3, 0xE3, 0x83, 0x88, 0xE3, 0x82, 0x99, 0x0D, + 0x52, 0xE3, 0x82, 0xA8, 0xE3, 0x82, 0xB9, 0xE3, + 0x82, 0xAF, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, + 0xE3, 0x82, 0x99, 0x0D, 0x52, 0xE3, 0x83, 0x95, + // Bytes 2f40 - 2f7f + 0xE3, 0x82, 0xA1, 0xE3, 0x83, 0xA9, 0xE3, 0x83, + 0x83, 0xE3, 0x83, 0x88, 0xE3, 0x82, 0x99, 0x0D, + 0x86, 0xE0, 0xB3, 0x86, 0xE0, 0xB3, 0x82, 0x01, + 0x86, 0xE0, 0xB7, 0x99, 0xE0, 0xB7, 0x8F, 0x01, + 0x03, 0x3C, 0xCC, 0xB8, 0x05, 0x03, 0x3D, 0xCC, + 0xB8, 0x05, 0x03, 0x3E, 0xCC, 0xB8, 0x05, 0x03, + 0x41, 0xCC, 0x80, 0xC9, 0x03, 0x41, 0xCC, 0x81, + 0xC9, 0x03, 0x41, 0xCC, 0x83, 0xC9, 0x03, 0x41, + // Bytes 2f80 - 2fbf + 0xCC, 0x84, 0xC9, 0x03, 0x41, 0xCC, 0x89, 0xC9, + 0x03, 0x41, 0xCC, 0x8C, 0xC9, 0x03, 0x41, 0xCC, + 0x8F, 0xC9, 0x03, 0x41, 0xCC, 0x91, 0xC9, 0x03, + 0x41, 0xCC, 0xA5, 0xB5, 0x03, 0x41, 0xCC, 0xA8, + 0xA5, 0x03, 0x42, 0xCC, 0x87, 0xC9, 0x03, 0x42, + 0xCC, 0xA3, 0xB5, 0x03, 0x42, 0xCC, 0xB1, 0xB5, + 0x03, 0x43, 0xCC, 0x81, 0xC9, 0x03, 0x43, 0xCC, + 0x82, 0xC9, 0x03, 0x43, 0xCC, 0x87, 0xC9, 0x03, + // Bytes 2fc0 - 2fff + 0x43, 0xCC, 0x8C, 0xC9, 0x03, 0x44, 0xCC, 0x87, + 0xC9, 0x03, 0x44, 0xCC, 0x8C, 0xC9, 0x03, 0x44, + 0xCC, 0xA3, 0xB5, 0x03, 0x44, 0xCC, 0xA7, 0xA5, + 0x03, 0x44, 0xCC, 0xAD, 0xB5, 0x03, 0x44, 0xCC, + 0xB1, 0xB5, 0x03, 0x45, 0xCC, 0x80, 0xC9, 0x03, + 0x45, 0xCC, 0x81, 0xC9, 0x03, 0x45, 0xCC, 0x83, + 0xC9, 0x03, 0x45, 0xCC, 0x86, 0xC9, 0x03, 0x45, + 0xCC, 0x87, 0xC9, 0x03, 0x45, 0xCC, 0x88, 0xC9, + // Bytes 3000 - 303f + 0x03, 0x45, 0xCC, 0x89, 0xC9, 0x03, 0x45, 0xCC, + 0x8C, 0xC9, 0x03, 0x45, 0xCC, 0x8F, 0xC9, 0x03, + 0x45, 0xCC, 0x91, 0xC9, 0x03, 0x45, 0xCC, 0xA8, + 0xA5, 0x03, 0x45, 0xCC, 0xAD, 0xB5, 0x03, 0x45, + 0xCC, 0xB0, 0xB5, 0x03, 0x46, 0xCC, 0x87, 0xC9, + 0x03, 0x47, 0xCC, 0x81, 0xC9, 0x03, 0x47, 0xCC, + 0x82, 0xC9, 0x03, 0x47, 0xCC, 0x84, 0xC9, 0x03, + 0x47, 0xCC, 0x86, 0xC9, 0x03, 0x47, 0xCC, 0x87, + // Bytes 3040 - 307f + 0xC9, 0x03, 0x47, 0xCC, 0x8C, 0xC9, 0x03, 0x47, + 0xCC, 0xA7, 0xA5, 0x03, 0x48, 0xCC, 0x82, 0xC9, + 0x03, 0x48, 0xCC, 0x87, 0xC9, 0x03, 0x48, 0xCC, + 0x88, 0xC9, 0x03, 0x48, 0xCC, 0x8C, 0xC9, 0x03, + 0x48, 0xCC, 0xA3, 0xB5, 0x03, 0x48, 0xCC, 0xA7, + 0xA5, 0x03, 0x48, 0xCC, 0xAE, 0xB5, 0x03, 0x49, + 0xCC, 0x80, 0xC9, 0x03, 0x49, 0xCC, 0x81, 0xC9, + 0x03, 0x49, 0xCC, 0x82, 0xC9, 0x03, 0x49, 0xCC, + // Bytes 3080 - 30bf + 0x83, 0xC9, 0x03, 0x49, 0xCC, 0x84, 0xC9, 0x03, + 0x49, 0xCC, 0x86, 0xC9, 0x03, 0x49, 0xCC, 0x87, + 0xC9, 0x03, 0x49, 0xCC, 0x89, 0xC9, 0x03, 0x49, + 0xCC, 0x8C, 0xC9, 0x03, 0x49, 0xCC, 0x8F, 0xC9, + 0x03, 0x49, 0xCC, 0x91, 0xC9, 0x03, 0x49, 0xCC, + 0xA3, 0xB5, 0x03, 0x49, 0xCC, 0xA8, 0xA5, 0x03, + 0x49, 0xCC, 0xB0, 0xB5, 0x03, 0x4A, 0xCC, 0x82, + 0xC9, 0x03, 0x4B, 0xCC, 0x81, 0xC9, 0x03, 0x4B, + // Bytes 30c0 - 30ff + 0xCC, 0x8C, 0xC9, 0x03, 0x4B, 0xCC, 0xA3, 0xB5, + 0x03, 0x4B, 0xCC, 0xA7, 0xA5, 0x03, 0x4B, 0xCC, + 0xB1, 0xB5, 0x03, 0x4C, 0xCC, 0x81, 0xC9, 0x03, + 0x4C, 0xCC, 0x8C, 0xC9, 0x03, 0x4C, 0xCC, 0xA7, + 0xA5, 0x03, 0x4C, 0xCC, 0xAD, 0xB5, 0x03, 0x4C, + 0xCC, 0xB1, 0xB5, 0x03, 0x4D, 0xCC, 0x81, 0xC9, + 0x03, 0x4D, 0xCC, 0x87, 0xC9, 0x03, 0x4D, 0xCC, + 0xA3, 0xB5, 0x03, 0x4E, 0xCC, 0x80, 0xC9, 0x03, + // Bytes 3100 - 313f + 0x4E, 0xCC, 0x81, 0xC9, 0x03, 0x4E, 0xCC, 0x83, + 0xC9, 0x03, 0x4E, 0xCC, 0x87, 0xC9, 0x03, 0x4E, + 0xCC, 0x8C, 0xC9, 0x03, 0x4E, 0xCC, 0xA3, 0xB5, + 0x03, 0x4E, 0xCC, 0xA7, 0xA5, 0x03, 0x4E, 0xCC, + 0xAD, 0xB5, 0x03, 0x4E, 0xCC, 0xB1, 0xB5, 0x03, + 0x4F, 0xCC, 0x80, 0xC9, 0x03, 0x4F, 0xCC, 0x81, + 0xC9, 0x03, 0x4F, 0xCC, 0x86, 0xC9, 0x03, 0x4F, + 0xCC, 0x89, 0xC9, 0x03, 0x4F, 0xCC, 0x8B, 0xC9, + // Bytes 3140 - 317f + 0x03, 0x4F, 0xCC, 0x8C, 0xC9, 0x03, 0x4F, 0xCC, + 0x8F, 0xC9, 0x03, 0x4F, 0xCC, 0x91, 0xC9, 0x03, + 0x50, 0xCC, 0x81, 0xC9, 0x03, 0x50, 0xCC, 0x87, + 0xC9, 0x03, 0x52, 0xCC, 0x81, 0xC9, 0x03, 0x52, + 0xCC, 0x87, 0xC9, 0x03, 0x52, 0xCC, 0x8C, 0xC9, + 0x03, 0x52, 0xCC, 0x8F, 0xC9, 0x03, 0x52, 0xCC, + 0x91, 0xC9, 0x03, 0x52, 0xCC, 0xA7, 0xA5, 0x03, + 0x52, 0xCC, 0xB1, 0xB5, 0x03, 0x53, 0xCC, 0x82, + // Bytes 3180 - 31bf + 0xC9, 0x03, 0x53, 0xCC, 0x87, 0xC9, 0x03, 0x53, + 0xCC, 0xA6, 0xB5, 0x03, 0x53, 0xCC, 0xA7, 0xA5, + 0x03, 0x54, 0xCC, 0x87, 0xC9, 0x03, 0x54, 0xCC, + 0x8C, 0xC9, 0x03, 0x54, 0xCC, 0xA3, 0xB5, 0x03, + 0x54, 0xCC, 0xA6, 0xB5, 0x03, 0x54, 0xCC, 0xA7, + 0xA5, 0x03, 0x54, 0xCC, 0xAD, 0xB5, 0x03, 0x54, + 0xCC, 0xB1, 0xB5, 0x03, 0x55, 0xCC, 0x80, 0xC9, + 0x03, 0x55, 0xCC, 0x81, 0xC9, 0x03, 0x55, 0xCC, + // Bytes 31c0 - 31ff + 0x82, 0xC9, 0x03, 0x55, 0xCC, 0x86, 0xC9, 0x03, + 0x55, 0xCC, 0x89, 0xC9, 0x03, 0x55, 0xCC, 0x8A, + 0xC9, 0x03, 0x55, 0xCC, 0x8B, 0xC9, 0x03, 0x55, + 0xCC, 0x8C, 0xC9, 0x03, 0x55, 0xCC, 0x8F, 0xC9, + 0x03, 0x55, 0xCC, 0x91, 0xC9, 0x03, 0x55, 0xCC, + 0xA3, 0xB5, 0x03, 0x55, 0xCC, 0xA4, 0xB5, 0x03, + 0x55, 0xCC, 0xA8, 0xA5, 0x03, 0x55, 0xCC, 0xAD, + 0xB5, 0x03, 0x55, 0xCC, 0xB0, 0xB5, 0x03, 0x56, + // Bytes 3200 - 323f + 0xCC, 0x83, 0xC9, 0x03, 0x56, 0xCC, 0xA3, 0xB5, + 0x03, 0x57, 0xCC, 0x80, 0xC9, 0x03, 0x57, 0xCC, + 0x81, 0xC9, 0x03, 0x57, 0xCC, 0x82, 0xC9, 0x03, + 0x57, 0xCC, 0x87, 0xC9, 0x03, 0x57, 0xCC, 0x88, + 0xC9, 0x03, 0x57, 0xCC, 0xA3, 0xB5, 0x03, 0x58, + 0xCC, 0x87, 0xC9, 0x03, 0x58, 0xCC, 0x88, 0xC9, + 0x03, 0x59, 0xCC, 0x80, 0xC9, 0x03, 0x59, 0xCC, + 0x81, 0xC9, 0x03, 0x59, 0xCC, 0x82, 0xC9, 0x03, + // Bytes 3240 - 327f + 0x59, 0xCC, 0x83, 0xC9, 0x03, 0x59, 0xCC, 0x84, + 0xC9, 0x03, 0x59, 0xCC, 0x87, 0xC9, 0x03, 0x59, + 0xCC, 0x88, 0xC9, 0x03, 0x59, 0xCC, 0x89, 0xC9, + 0x03, 0x59, 0xCC, 0xA3, 0xB5, 0x03, 0x5A, 0xCC, + 0x81, 0xC9, 0x03, 0x5A, 0xCC, 0x82, 0xC9, 0x03, + 0x5A, 0xCC, 0x87, 0xC9, 0x03, 0x5A, 0xCC, 0x8C, + 0xC9, 0x03, 0x5A, 0xCC, 0xA3, 0xB5, 0x03, 0x5A, + 0xCC, 0xB1, 0xB5, 0x03, 0x61, 0xCC, 0x80, 0xC9, + // Bytes 3280 - 32bf + 0x03, 0x61, 0xCC, 0x81, 0xC9, 0x03, 0x61, 0xCC, + 0x83, 0xC9, 0x03, 0x61, 0xCC, 0x84, 0xC9, 0x03, + 0x61, 0xCC, 0x89, 0xC9, 0x03, 0x61, 0xCC, 0x8C, + 0xC9, 0x03, 0x61, 0xCC, 0x8F, 0xC9, 0x03, 0x61, + 0xCC, 0x91, 0xC9, 0x03, 0x61, 0xCC, 0xA5, 0xB5, + 0x03, 0x61, 0xCC, 0xA8, 0xA5, 0x03, 0x62, 0xCC, + 0x87, 0xC9, 0x03, 0x62, 0xCC, 0xA3, 0xB5, 0x03, + 0x62, 0xCC, 0xB1, 0xB5, 0x03, 0x63, 0xCC, 0x81, + // Bytes 32c0 - 32ff + 0xC9, 0x03, 0x63, 0xCC, 0x82, 0xC9, 0x03, 0x63, + 0xCC, 0x87, 0xC9, 0x03, 0x63, 0xCC, 0x8C, 0xC9, + 0x03, 0x64, 0xCC, 0x87, 0xC9, 0x03, 0x64, 0xCC, + 0x8C, 0xC9, 0x03, 0x64, 0xCC, 0xA3, 0xB5, 0x03, + 0x64, 0xCC, 0xA7, 0xA5, 0x03, 0x64, 0xCC, 0xAD, + 0xB5, 0x03, 0x64, 0xCC, 0xB1, 0xB5, 0x03, 0x65, + 0xCC, 0x80, 0xC9, 0x03, 0x65, 0xCC, 0x81, 0xC9, + 0x03, 0x65, 0xCC, 0x83, 0xC9, 0x03, 0x65, 0xCC, + // Bytes 3300 - 333f + 0x86, 0xC9, 0x03, 0x65, 0xCC, 0x87, 0xC9, 0x03, + 0x65, 0xCC, 0x88, 0xC9, 0x03, 0x65, 0xCC, 0x89, + 0xC9, 0x03, 0x65, 0xCC, 0x8C, 0xC9, 0x03, 0x65, + 0xCC, 0x8F, 0xC9, 0x03, 0x65, 0xCC, 0x91, 0xC9, + 0x03, 0x65, 0xCC, 0xA8, 0xA5, 0x03, 0x65, 0xCC, + 0xAD, 0xB5, 0x03, 0x65, 0xCC, 0xB0, 0xB5, 0x03, + 0x66, 0xCC, 0x87, 0xC9, 0x03, 0x67, 0xCC, 0x81, + 0xC9, 0x03, 0x67, 0xCC, 0x82, 0xC9, 0x03, 0x67, + // Bytes 3340 - 337f + 0xCC, 0x84, 0xC9, 0x03, 0x67, 0xCC, 0x86, 0xC9, + 0x03, 0x67, 0xCC, 0x87, 0xC9, 0x03, 0x67, 0xCC, + 0x8C, 0xC9, 0x03, 0x67, 0xCC, 0xA7, 0xA5, 0x03, + 0x68, 0xCC, 0x82, 0xC9, 0x03, 0x68, 0xCC, 0x87, + 0xC9, 0x03, 0x68, 0xCC, 0x88, 0xC9, 0x03, 0x68, + 0xCC, 0x8C, 0xC9, 0x03, 0x68, 0xCC, 0xA3, 0xB5, + 0x03, 0x68, 0xCC, 0xA7, 0xA5, 0x03, 0x68, 0xCC, + 0xAE, 0xB5, 0x03, 0x68, 0xCC, 0xB1, 0xB5, 0x03, + // Bytes 3380 - 33bf + 0x69, 0xCC, 0x80, 0xC9, 0x03, 0x69, 0xCC, 0x81, + 0xC9, 0x03, 0x69, 0xCC, 0x82, 0xC9, 0x03, 0x69, + 0xCC, 0x83, 0xC9, 0x03, 0x69, 0xCC, 0x84, 0xC9, + 0x03, 0x69, 0xCC, 0x86, 0xC9, 0x03, 0x69, 0xCC, + 0x89, 0xC9, 0x03, 0x69, 0xCC, 0x8C, 0xC9, 0x03, + 0x69, 0xCC, 0x8F, 0xC9, 0x03, 0x69, 0xCC, 0x91, + 0xC9, 0x03, 0x69, 0xCC, 0xA3, 0xB5, 0x03, 0x69, + 0xCC, 0xA8, 0xA5, 0x03, 0x69, 0xCC, 0xB0, 0xB5, + // Bytes 33c0 - 33ff + 0x03, 0x6A, 0xCC, 0x82, 0xC9, 0x03, 0x6A, 0xCC, + 0x8C, 0xC9, 0x03, 0x6B, 0xCC, 0x81, 0xC9, 0x03, + 0x6B, 0xCC, 0x8C, 0xC9, 0x03, 0x6B, 0xCC, 0xA3, + 0xB5, 0x03, 0x6B, 0xCC, 0xA7, 0xA5, 0x03, 0x6B, + 0xCC, 0xB1, 0xB5, 0x03, 0x6C, 0xCC, 0x81, 0xC9, + 0x03, 0x6C, 0xCC, 0x8C, 0xC9, 0x03, 0x6C, 0xCC, + 0xA7, 0xA5, 0x03, 0x6C, 0xCC, 0xAD, 0xB5, 0x03, + 0x6C, 0xCC, 0xB1, 0xB5, 0x03, 0x6D, 0xCC, 0x81, + // Bytes 3400 - 343f + 0xC9, 0x03, 0x6D, 0xCC, 0x87, 0xC9, 0x03, 0x6D, + 0xCC, 0xA3, 0xB5, 0x03, 0x6E, 0xCC, 0x80, 0xC9, + 0x03, 0x6E, 0xCC, 0x81, 0xC9, 0x03, 0x6E, 0xCC, + 0x83, 0xC9, 0x03, 0x6E, 0xCC, 0x87, 0xC9, 0x03, + 0x6E, 0xCC, 0x8C, 0xC9, 0x03, 0x6E, 0xCC, 0xA3, + 0xB5, 0x03, 0x6E, 0xCC, 0xA7, 0xA5, 0x03, 0x6E, + 0xCC, 0xAD, 0xB5, 0x03, 0x6E, 0xCC, 0xB1, 0xB5, + 0x03, 0x6F, 0xCC, 0x80, 0xC9, 0x03, 0x6F, 0xCC, + // Bytes 3440 - 347f + 0x81, 0xC9, 0x03, 0x6F, 0xCC, 0x86, 0xC9, 0x03, + 0x6F, 0xCC, 0x89, 0xC9, 0x03, 0x6F, 0xCC, 0x8B, + 0xC9, 0x03, 0x6F, 0xCC, 0x8C, 0xC9, 0x03, 0x6F, + 0xCC, 0x8F, 0xC9, 0x03, 0x6F, 0xCC, 0x91, 0xC9, + 0x03, 0x70, 0xCC, 0x81, 0xC9, 0x03, 0x70, 0xCC, + 0x87, 0xC9, 0x03, 0x72, 0xCC, 0x81, 0xC9, 0x03, + 0x72, 0xCC, 0x87, 0xC9, 0x03, 0x72, 0xCC, 0x8C, + 0xC9, 0x03, 0x72, 0xCC, 0x8F, 0xC9, 0x03, 0x72, + // Bytes 3480 - 34bf + 0xCC, 0x91, 0xC9, 0x03, 0x72, 0xCC, 0xA7, 0xA5, + 0x03, 0x72, 0xCC, 0xB1, 0xB5, 0x03, 0x73, 0xCC, + 0x82, 0xC9, 0x03, 0x73, 0xCC, 0x87, 0xC9, 0x03, + 0x73, 0xCC, 0xA6, 0xB5, 0x03, 0x73, 0xCC, 0xA7, + 0xA5, 0x03, 0x74, 0xCC, 0x87, 0xC9, 0x03, 0x74, + 0xCC, 0x88, 0xC9, 0x03, 0x74, 0xCC, 0x8C, 0xC9, + 0x03, 0x74, 0xCC, 0xA3, 0xB5, 0x03, 0x74, 0xCC, + 0xA6, 0xB5, 0x03, 0x74, 0xCC, 0xA7, 0xA5, 0x03, + // Bytes 34c0 - 34ff + 0x74, 0xCC, 0xAD, 0xB5, 0x03, 0x74, 0xCC, 0xB1, + 0xB5, 0x03, 0x75, 0xCC, 0x80, 0xC9, 0x03, 0x75, + 0xCC, 0x81, 0xC9, 0x03, 0x75, 0xCC, 0x82, 0xC9, + 0x03, 0x75, 0xCC, 0x86, 0xC9, 0x03, 0x75, 0xCC, + 0x89, 0xC9, 0x03, 0x75, 0xCC, 0x8A, 0xC9, 0x03, + 0x75, 0xCC, 0x8B, 0xC9, 0x03, 0x75, 0xCC, 0x8C, + 0xC9, 0x03, 0x75, 0xCC, 0x8F, 0xC9, 0x03, 0x75, + 0xCC, 0x91, 0xC9, 0x03, 0x75, 0xCC, 0xA3, 0xB5, + // Bytes 3500 - 353f + 0x03, 0x75, 0xCC, 0xA4, 0xB5, 0x03, 0x75, 0xCC, + 0xA8, 0xA5, 0x03, 0x75, 0xCC, 0xAD, 0xB5, 0x03, + 0x75, 0xCC, 0xB0, 0xB5, 0x03, 0x76, 0xCC, 0x83, + 0xC9, 0x03, 0x76, 0xCC, 0xA3, 0xB5, 0x03, 0x77, + 0xCC, 0x80, 0xC9, 0x03, 0x77, 0xCC, 0x81, 0xC9, + 0x03, 0x77, 0xCC, 0x82, 0xC9, 0x03, 0x77, 0xCC, + 0x87, 0xC9, 0x03, 0x77, 0xCC, 0x88, 0xC9, 0x03, + 0x77, 0xCC, 0x8A, 0xC9, 0x03, 0x77, 0xCC, 0xA3, + // Bytes 3540 - 357f + 0xB5, 0x03, 0x78, 0xCC, 0x87, 0xC9, 0x03, 0x78, + 0xCC, 0x88, 0xC9, 0x03, 0x79, 0xCC, 0x80, 0xC9, + 0x03, 0x79, 0xCC, 0x81, 0xC9, 0x03, 0x79, 0xCC, + 0x82, 0xC9, 0x03, 0x79, 0xCC, 0x83, 0xC9, 0x03, + 0x79, 0xCC, 0x84, 0xC9, 0x03, 0x79, 0xCC, 0x87, + 0xC9, 0x03, 0x79, 0xCC, 0x88, 0xC9, 0x03, 0x79, + 0xCC, 0x89, 0xC9, 0x03, 0x79, 0xCC, 0x8A, 0xC9, + 0x03, 0x79, 0xCC, 0xA3, 0xB5, 0x03, 0x7A, 0xCC, + // Bytes 3580 - 35bf + 0x81, 0xC9, 0x03, 0x7A, 0xCC, 0x82, 0xC9, 0x03, + 0x7A, 0xCC, 0x87, 0xC9, 0x03, 0x7A, 0xCC, 0x8C, + 0xC9, 0x03, 0x7A, 0xCC, 0xA3, 0xB5, 0x03, 0x7A, + 0xCC, 0xB1, 0xB5, 0x04, 0xC2, 0xA8, 0xCC, 0x80, + 0xCA, 0x04, 0xC2, 0xA8, 0xCC, 0x81, 0xCA, 0x04, + 0xC2, 0xA8, 0xCD, 0x82, 0xCA, 0x04, 0xC3, 0x86, + 0xCC, 0x81, 0xC9, 0x04, 0xC3, 0x86, 0xCC, 0x84, + 0xC9, 0x04, 0xC3, 0x98, 0xCC, 0x81, 0xC9, 0x04, + // Bytes 35c0 - 35ff + 0xC3, 0xA6, 0xCC, 0x81, 0xC9, 0x04, 0xC3, 0xA6, + 0xCC, 0x84, 0xC9, 0x04, 0xC3, 0xB8, 0xCC, 0x81, + 0xC9, 0x04, 0xC5, 0xBF, 0xCC, 0x87, 0xC9, 0x04, + 0xC6, 0xB7, 0xCC, 0x8C, 0xC9, 0x04, 0xCA, 0x92, + 0xCC, 0x8C, 0xC9, 0x04, 0xCE, 0x91, 0xCC, 0x80, + 0xC9, 0x04, 0xCE, 0x91, 0xCC, 0x81, 0xC9, 0x04, + 0xCE, 0x91, 0xCC, 0x84, 0xC9, 0x04, 0xCE, 0x91, + 0xCC, 0x86, 0xC9, 0x04, 0xCE, 0x91, 0xCD, 0x85, + // Bytes 3600 - 363f + 0xD9, 0x04, 0xCE, 0x95, 0xCC, 0x80, 0xC9, 0x04, + 0xCE, 0x95, 0xCC, 0x81, 0xC9, 0x04, 0xCE, 0x97, + 0xCC, 0x80, 0xC9, 0x04, 0xCE, 0x97, 0xCC, 0x81, + 0xC9, 0x04, 0xCE, 0x97, 0xCD, 0x85, 0xD9, 0x04, + 0xCE, 0x99, 0xCC, 0x80, 0xC9, 0x04, 0xCE, 0x99, + 0xCC, 0x81, 0xC9, 0x04, 0xCE, 0x99, 0xCC, 0x84, + 0xC9, 0x04, 0xCE, 0x99, 0xCC, 0x86, 0xC9, 0x04, + 0xCE, 0x99, 0xCC, 0x88, 0xC9, 0x04, 0xCE, 0x9F, + // Bytes 3640 - 367f + 0xCC, 0x80, 0xC9, 0x04, 0xCE, 0x9F, 0xCC, 0x81, + 0xC9, 0x04, 0xCE, 0xA1, 0xCC, 0x94, 0xC9, 0x04, + 0xCE, 0xA5, 0xCC, 0x80, 0xC9, 0x04, 0xCE, 0xA5, + 0xCC, 0x81, 0xC9, 0x04, 0xCE, 0xA5, 0xCC, 0x84, + 0xC9, 0x04, 0xCE, 0xA5, 0xCC, 0x86, 0xC9, 0x04, + 0xCE, 0xA5, 0xCC, 0x88, 0xC9, 0x04, 0xCE, 0xA9, + 0xCC, 0x80, 0xC9, 0x04, 0xCE, 0xA9, 0xCC, 0x81, + 0xC9, 0x04, 0xCE, 0xA9, 0xCD, 0x85, 0xD9, 0x04, + // Bytes 3680 - 36bf + 0xCE, 0xB1, 0xCC, 0x84, 0xC9, 0x04, 0xCE, 0xB1, + 0xCC, 0x86, 0xC9, 0x04, 0xCE, 0xB1, 0xCD, 0x85, + 0xD9, 0x04, 0xCE, 0xB5, 0xCC, 0x80, 0xC9, 0x04, + 0xCE, 0xB5, 0xCC, 0x81, 0xC9, 0x04, 0xCE, 0xB7, + 0xCD, 0x85, 0xD9, 0x04, 0xCE, 0xB9, 0xCC, 0x80, + 0xC9, 0x04, 0xCE, 0xB9, 0xCC, 0x81, 0xC9, 0x04, + 0xCE, 0xB9, 0xCC, 0x84, 0xC9, 0x04, 0xCE, 0xB9, + 0xCC, 0x86, 0xC9, 0x04, 0xCE, 0xB9, 0xCD, 0x82, + // Bytes 36c0 - 36ff + 0xC9, 0x04, 0xCE, 0xBF, 0xCC, 0x80, 0xC9, 0x04, + 0xCE, 0xBF, 0xCC, 0x81, 0xC9, 0x04, 0xCF, 0x81, + 0xCC, 0x93, 0xC9, 0x04, 0xCF, 0x81, 0xCC, 0x94, + 0xC9, 0x04, 0xCF, 0x85, 0xCC, 0x80, 0xC9, 0x04, + 0xCF, 0x85, 0xCC, 0x81, 0xC9, 0x04, 0xCF, 0x85, + 0xCC, 0x84, 0xC9, 0x04, 0xCF, 0x85, 0xCC, 0x86, + 0xC9, 0x04, 0xCF, 0x85, 0xCD, 0x82, 0xC9, 0x04, + 0xCF, 0x89, 0xCD, 0x85, 0xD9, 0x04, 0xCF, 0x92, + // Bytes 3700 - 373f + 0xCC, 0x81, 0xC9, 0x04, 0xCF, 0x92, 0xCC, 0x88, + 0xC9, 0x04, 0xD0, 0x86, 0xCC, 0x88, 0xC9, 0x04, + 0xD0, 0x90, 0xCC, 0x86, 0xC9, 0x04, 0xD0, 0x90, + 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0x93, 0xCC, 0x81, + 0xC9, 0x04, 0xD0, 0x95, 0xCC, 0x80, 0xC9, 0x04, + 0xD0, 0x95, 0xCC, 0x86, 0xC9, 0x04, 0xD0, 0x95, + 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0x96, 0xCC, 0x86, + 0xC9, 0x04, 0xD0, 0x96, 0xCC, 0x88, 0xC9, 0x04, + // Bytes 3740 - 377f + 0xD0, 0x97, 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0x98, + 0xCC, 0x80, 0xC9, 0x04, 0xD0, 0x98, 0xCC, 0x84, + 0xC9, 0x04, 0xD0, 0x98, 0xCC, 0x86, 0xC9, 0x04, + 0xD0, 0x98, 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0x9A, + 0xCC, 0x81, 0xC9, 0x04, 0xD0, 0x9E, 0xCC, 0x88, + 0xC9, 0x04, 0xD0, 0xA3, 0xCC, 0x84, 0xC9, 0x04, + 0xD0, 0xA3, 0xCC, 0x86, 0xC9, 0x04, 0xD0, 0xA3, + 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0xA3, 0xCC, 0x8B, + // Bytes 3780 - 37bf + 0xC9, 0x04, 0xD0, 0xA7, 0xCC, 0x88, 0xC9, 0x04, + 0xD0, 0xAB, 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0xAD, + 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0xB0, 0xCC, 0x86, + 0xC9, 0x04, 0xD0, 0xB0, 0xCC, 0x88, 0xC9, 0x04, + 0xD0, 0xB3, 0xCC, 0x81, 0xC9, 0x04, 0xD0, 0xB5, + 0xCC, 0x80, 0xC9, 0x04, 0xD0, 0xB5, 0xCC, 0x86, + 0xC9, 0x04, 0xD0, 0xB5, 0xCC, 0x88, 0xC9, 0x04, + 0xD0, 0xB6, 0xCC, 0x86, 0xC9, 0x04, 0xD0, 0xB6, + // Bytes 37c0 - 37ff + 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0xB7, 0xCC, 0x88, + 0xC9, 0x04, 0xD0, 0xB8, 0xCC, 0x80, 0xC9, 0x04, + 0xD0, 0xB8, 0xCC, 0x84, 0xC9, 0x04, 0xD0, 0xB8, + 0xCC, 0x86, 0xC9, 0x04, 0xD0, 0xB8, 0xCC, 0x88, + 0xC9, 0x04, 0xD0, 0xBA, 0xCC, 0x81, 0xC9, 0x04, + 0xD0, 0xBE, 0xCC, 0x88, 0xC9, 0x04, 0xD1, 0x83, + 0xCC, 0x84, 0xC9, 0x04, 0xD1, 0x83, 0xCC, 0x86, + 0xC9, 0x04, 0xD1, 0x83, 0xCC, 0x88, 0xC9, 0x04, + // Bytes 3800 - 383f + 0xD1, 0x83, 0xCC, 0x8B, 0xC9, 0x04, 0xD1, 0x87, + 0xCC, 0x88, 0xC9, 0x04, 0xD1, 0x8B, 0xCC, 0x88, + 0xC9, 0x04, 0xD1, 0x8D, 0xCC, 0x88, 0xC9, 0x04, + 0xD1, 0x96, 0xCC, 0x88, 0xC9, 0x04, 0xD1, 0xB4, + 0xCC, 0x8F, 0xC9, 0x04, 0xD1, 0xB5, 0xCC, 0x8F, + 0xC9, 0x04, 0xD3, 0x98, 0xCC, 0x88, 0xC9, 0x04, + 0xD3, 0x99, 0xCC, 0x88, 0xC9, 0x04, 0xD3, 0xA8, + 0xCC, 0x88, 0xC9, 0x04, 0xD3, 0xA9, 0xCC, 0x88, + // Bytes 3840 - 387f + 0xC9, 0x04, 0xD8, 0xA7, 0xD9, 0x93, 0xC9, 0x04, + 0xD8, 0xA7, 0xD9, 0x94, 0xC9, 0x04, 0xD8, 0xA7, + 0xD9, 0x95, 0xB5, 0x04, 0xD9, 0x88, 0xD9, 0x94, + 0xC9, 0x04, 0xD9, 0x8A, 0xD9, 0x94, 0xC9, 0x04, + 0xDB, 0x81, 0xD9, 0x94, 0xC9, 0x04, 0xDB, 0x92, + 0xD9, 0x94, 0xC9, 0x04, 0xDB, 0x95, 0xD9, 0x94, + 0xC9, 0x05, 0x41, 0xCC, 0x82, 0xCC, 0x80, 0xCA, + 0x05, 0x41, 0xCC, 0x82, 0xCC, 0x81, 0xCA, 0x05, + // Bytes 3880 - 38bf + 0x41, 0xCC, 0x82, 0xCC, 0x83, 0xCA, 0x05, 0x41, + 0xCC, 0x82, 0xCC, 0x89, 0xCA, 0x05, 0x41, 0xCC, + 0x86, 0xCC, 0x80, 0xCA, 0x05, 0x41, 0xCC, 0x86, + 0xCC, 0x81, 0xCA, 0x05, 0x41, 0xCC, 0x86, 0xCC, + 0x83, 0xCA, 0x05, 0x41, 0xCC, 0x86, 0xCC, 0x89, + 0xCA, 0x05, 0x41, 0xCC, 0x87, 0xCC, 0x84, 0xCA, + 0x05, 0x41, 0xCC, 0x88, 0xCC, 0x84, 0xCA, 0x05, + 0x41, 0xCC, 0x8A, 0xCC, 0x81, 0xCA, 0x05, 0x41, + // Bytes 38c0 - 38ff + 0xCC, 0xA3, 0xCC, 0x82, 0xCA, 0x05, 0x41, 0xCC, + 0xA3, 0xCC, 0x86, 0xCA, 0x05, 0x43, 0xCC, 0xA7, + 0xCC, 0x81, 0xCA, 0x05, 0x45, 0xCC, 0x82, 0xCC, + 0x80, 0xCA, 0x05, 0x45, 0xCC, 0x82, 0xCC, 0x81, + 0xCA, 0x05, 0x45, 0xCC, 0x82, 0xCC, 0x83, 0xCA, + 0x05, 0x45, 0xCC, 0x82, 0xCC, 0x89, 0xCA, 0x05, + 0x45, 0xCC, 0x84, 0xCC, 0x80, 0xCA, 0x05, 0x45, + 0xCC, 0x84, 0xCC, 0x81, 0xCA, 0x05, 0x45, 0xCC, + // Bytes 3900 - 393f + 0xA3, 0xCC, 0x82, 0xCA, 0x05, 0x45, 0xCC, 0xA7, + 0xCC, 0x86, 0xCA, 0x05, 0x49, 0xCC, 0x88, 0xCC, + 0x81, 0xCA, 0x05, 0x4C, 0xCC, 0xA3, 0xCC, 0x84, + 0xCA, 0x05, 0x4F, 0xCC, 0x82, 0xCC, 0x80, 0xCA, + 0x05, 0x4F, 0xCC, 0x82, 0xCC, 0x81, 0xCA, 0x05, + 0x4F, 0xCC, 0x82, 0xCC, 0x83, 0xCA, 0x05, 0x4F, + 0xCC, 0x82, 0xCC, 0x89, 0xCA, 0x05, 0x4F, 0xCC, + 0x83, 0xCC, 0x81, 0xCA, 0x05, 0x4F, 0xCC, 0x83, + // Bytes 3940 - 397f + 0xCC, 0x84, 0xCA, 0x05, 0x4F, 0xCC, 0x83, 0xCC, + 0x88, 0xCA, 0x05, 0x4F, 0xCC, 0x84, 0xCC, 0x80, + 0xCA, 0x05, 0x4F, 0xCC, 0x84, 0xCC, 0x81, 0xCA, + 0x05, 0x4F, 0xCC, 0x87, 0xCC, 0x84, 0xCA, 0x05, + 0x4F, 0xCC, 0x88, 0xCC, 0x84, 0xCA, 0x05, 0x4F, + 0xCC, 0x9B, 0xCC, 0x80, 0xCA, 0x05, 0x4F, 0xCC, + 0x9B, 0xCC, 0x81, 0xCA, 0x05, 0x4F, 0xCC, 0x9B, + 0xCC, 0x83, 0xCA, 0x05, 0x4F, 0xCC, 0x9B, 0xCC, + // Bytes 3980 - 39bf + 0x89, 0xCA, 0x05, 0x4F, 0xCC, 0x9B, 0xCC, 0xA3, + 0xB6, 0x05, 0x4F, 0xCC, 0xA3, 0xCC, 0x82, 0xCA, + 0x05, 0x4F, 0xCC, 0xA8, 0xCC, 0x84, 0xCA, 0x05, + 0x52, 0xCC, 0xA3, 0xCC, 0x84, 0xCA, 0x05, 0x53, + 0xCC, 0x81, 0xCC, 0x87, 0xCA, 0x05, 0x53, 0xCC, + 0x8C, 0xCC, 0x87, 0xCA, 0x05, 0x53, 0xCC, 0xA3, + 0xCC, 0x87, 0xCA, 0x05, 0x55, 0xCC, 0x83, 0xCC, + 0x81, 0xCA, 0x05, 0x55, 0xCC, 0x84, 0xCC, 0x88, + // Bytes 39c0 - 39ff + 0xCA, 0x05, 0x55, 0xCC, 0x88, 0xCC, 0x80, 0xCA, + 0x05, 0x55, 0xCC, 0x88, 0xCC, 0x81, 0xCA, 0x05, + 0x55, 0xCC, 0x88, 0xCC, 0x84, 0xCA, 0x05, 0x55, + 0xCC, 0x88, 0xCC, 0x8C, 0xCA, 0x05, 0x55, 0xCC, + 0x9B, 0xCC, 0x80, 0xCA, 0x05, 0x55, 0xCC, 0x9B, + 0xCC, 0x81, 0xCA, 0x05, 0x55, 0xCC, 0x9B, 0xCC, + 0x83, 0xCA, 0x05, 0x55, 0xCC, 0x9B, 0xCC, 0x89, + 0xCA, 0x05, 0x55, 0xCC, 0x9B, 0xCC, 0xA3, 0xB6, + // Bytes 3a00 - 3a3f + 0x05, 0x61, 0xCC, 0x82, 0xCC, 0x80, 0xCA, 0x05, + 0x61, 0xCC, 0x82, 0xCC, 0x81, 0xCA, 0x05, 0x61, + 0xCC, 0x82, 0xCC, 0x83, 0xCA, 0x05, 0x61, 0xCC, + 0x82, 0xCC, 0x89, 0xCA, 0x05, 0x61, 0xCC, 0x86, + 0xCC, 0x80, 0xCA, 0x05, 0x61, 0xCC, 0x86, 0xCC, + 0x81, 0xCA, 0x05, 0x61, 0xCC, 0x86, 0xCC, 0x83, + 0xCA, 0x05, 0x61, 0xCC, 0x86, 0xCC, 0x89, 0xCA, + 0x05, 0x61, 0xCC, 0x87, 0xCC, 0x84, 0xCA, 0x05, + // Bytes 3a40 - 3a7f + 0x61, 0xCC, 0x88, 0xCC, 0x84, 0xCA, 0x05, 0x61, + 0xCC, 0x8A, 0xCC, 0x81, 0xCA, 0x05, 0x61, 0xCC, + 0xA3, 0xCC, 0x82, 0xCA, 0x05, 0x61, 0xCC, 0xA3, + 0xCC, 0x86, 0xCA, 0x05, 0x63, 0xCC, 0xA7, 0xCC, + 0x81, 0xCA, 0x05, 0x65, 0xCC, 0x82, 0xCC, 0x80, + 0xCA, 0x05, 0x65, 0xCC, 0x82, 0xCC, 0x81, 0xCA, + 0x05, 0x65, 0xCC, 0x82, 0xCC, 0x83, 0xCA, 0x05, + 0x65, 0xCC, 0x82, 0xCC, 0x89, 0xCA, 0x05, 0x65, + // Bytes 3a80 - 3abf + 0xCC, 0x84, 0xCC, 0x80, 0xCA, 0x05, 0x65, 0xCC, + 0x84, 0xCC, 0x81, 0xCA, 0x05, 0x65, 0xCC, 0xA3, + 0xCC, 0x82, 0xCA, 0x05, 0x65, 0xCC, 0xA7, 0xCC, + 0x86, 0xCA, 0x05, 0x69, 0xCC, 0x88, 0xCC, 0x81, + 0xCA, 0x05, 0x6C, 0xCC, 0xA3, 0xCC, 0x84, 0xCA, + 0x05, 0x6F, 0xCC, 0x82, 0xCC, 0x80, 0xCA, 0x05, + 0x6F, 0xCC, 0x82, 0xCC, 0x81, 0xCA, 0x05, 0x6F, + 0xCC, 0x82, 0xCC, 0x83, 0xCA, 0x05, 0x6F, 0xCC, + // Bytes 3ac0 - 3aff + 0x82, 0xCC, 0x89, 0xCA, 0x05, 0x6F, 0xCC, 0x83, + 0xCC, 0x81, 0xCA, 0x05, 0x6F, 0xCC, 0x83, 0xCC, + 0x84, 0xCA, 0x05, 0x6F, 0xCC, 0x83, 0xCC, 0x88, + 0xCA, 0x05, 0x6F, 0xCC, 0x84, 0xCC, 0x80, 0xCA, + 0x05, 0x6F, 0xCC, 0x84, 0xCC, 0x81, 0xCA, 0x05, + 0x6F, 0xCC, 0x87, 0xCC, 0x84, 0xCA, 0x05, 0x6F, + 0xCC, 0x88, 0xCC, 0x84, 0xCA, 0x05, 0x6F, 0xCC, + 0x9B, 0xCC, 0x80, 0xCA, 0x05, 0x6F, 0xCC, 0x9B, + // Bytes 3b00 - 3b3f + 0xCC, 0x81, 0xCA, 0x05, 0x6F, 0xCC, 0x9B, 0xCC, + 0x83, 0xCA, 0x05, 0x6F, 0xCC, 0x9B, 0xCC, 0x89, + 0xCA, 0x05, 0x6F, 0xCC, 0x9B, 0xCC, 0xA3, 0xB6, + 0x05, 0x6F, 0xCC, 0xA3, 0xCC, 0x82, 0xCA, 0x05, + 0x6F, 0xCC, 0xA8, 0xCC, 0x84, 0xCA, 0x05, 0x72, + 0xCC, 0xA3, 0xCC, 0x84, 0xCA, 0x05, 0x73, 0xCC, + 0x81, 0xCC, 0x87, 0xCA, 0x05, 0x73, 0xCC, 0x8C, + 0xCC, 0x87, 0xCA, 0x05, 0x73, 0xCC, 0xA3, 0xCC, + // Bytes 3b40 - 3b7f + 0x87, 0xCA, 0x05, 0x75, 0xCC, 0x83, 0xCC, 0x81, + 0xCA, 0x05, 0x75, 0xCC, 0x84, 0xCC, 0x88, 0xCA, + 0x05, 0x75, 0xCC, 0x88, 0xCC, 0x80, 0xCA, 0x05, + 0x75, 0xCC, 0x88, 0xCC, 0x81, 0xCA, 0x05, 0x75, + 0xCC, 0x88, 0xCC, 0x84, 0xCA, 0x05, 0x75, 0xCC, + 0x88, 0xCC, 0x8C, 0xCA, 0x05, 0x75, 0xCC, 0x9B, + 0xCC, 0x80, 0xCA, 0x05, 0x75, 0xCC, 0x9B, 0xCC, + 0x81, 0xCA, 0x05, 0x75, 0xCC, 0x9B, 0xCC, 0x83, + // Bytes 3b80 - 3bbf + 0xCA, 0x05, 0x75, 0xCC, 0x9B, 0xCC, 0x89, 0xCA, + 0x05, 0x75, 0xCC, 0x9B, 0xCC, 0xA3, 0xB6, 0x05, + 0xE1, 0xBE, 0xBF, 0xCC, 0x80, 0xCA, 0x05, 0xE1, + 0xBE, 0xBF, 0xCC, 0x81, 0xCA, 0x05, 0xE1, 0xBE, + 0xBF, 0xCD, 0x82, 0xCA, 0x05, 0xE1, 0xBF, 0xBE, + 0xCC, 0x80, 0xCA, 0x05, 0xE1, 0xBF, 0xBE, 0xCC, + 0x81, 0xCA, 0x05, 0xE1, 0xBF, 0xBE, 0xCD, 0x82, + 0xCA, 0x05, 0xE2, 0x86, 0x90, 0xCC, 0xB8, 0x05, + // Bytes 3bc0 - 3bff + 0x05, 0xE2, 0x86, 0x92, 0xCC, 0xB8, 0x05, 0x05, + 0xE2, 0x86, 0x94, 0xCC, 0xB8, 0x05, 0x05, 0xE2, + 0x87, 0x90, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x87, + 0x92, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x87, 0x94, + 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x88, 0x83, 0xCC, + 0xB8, 0x05, 0x05, 0xE2, 0x88, 0x88, 0xCC, 0xB8, + 0x05, 0x05, 0xE2, 0x88, 0x8B, 0xCC, 0xB8, 0x05, + 0x05, 0xE2, 0x88, 0xA3, 0xCC, 0xB8, 0x05, 0x05, + // Bytes 3c00 - 3c3f + 0xE2, 0x88, 0xA5, 0xCC, 0xB8, 0x05, 0x05, 0xE2, + 0x88, 0xBC, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, + 0x83, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, 0x85, + 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, 0x88, 0xCC, + 0xB8, 0x05, 0x05, 0xE2, 0x89, 0x8D, 0xCC, 0xB8, + 0x05, 0x05, 0xE2, 0x89, 0xA1, 0xCC, 0xB8, 0x05, + 0x05, 0xE2, 0x89, 0xA4, 0xCC, 0xB8, 0x05, 0x05, + 0xE2, 0x89, 0xA5, 0xCC, 0xB8, 0x05, 0x05, 0xE2, + // Bytes 3c40 - 3c7f + 0x89, 0xB2, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, + 0xB3, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, 0xB6, + 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, 0xB7, 0xCC, + 0xB8, 0x05, 0x05, 0xE2, 0x89, 0xBA, 0xCC, 0xB8, + 0x05, 0x05, 0xE2, 0x89, 0xBB, 0xCC, 0xB8, 0x05, + 0x05, 0xE2, 0x89, 0xBC, 0xCC, 0xB8, 0x05, 0x05, + 0xE2, 0x89, 0xBD, 0xCC, 0xB8, 0x05, 0x05, 0xE2, + 0x8A, 0x82, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x8A, + // Bytes 3c80 - 3cbf + 0x83, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x8A, 0x86, + 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x8A, 0x87, 0xCC, + 0xB8, 0x05, 0x05, 0xE2, 0x8A, 0x91, 0xCC, 0xB8, + 0x05, 0x05, 0xE2, 0x8A, 0x92, 0xCC, 0xB8, 0x05, + 0x05, 0xE2, 0x8A, 0xA2, 0xCC, 0xB8, 0x05, 0x05, + 0xE2, 0x8A, 0xA8, 0xCC, 0xB8, 0x05, 0x05, 0xE2, + 0x8A, 0xA9, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x8A, + 0xAB, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x8A, 0xB2, + // Bytes 3cc0 - 3cff + 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x8A, 0xB3, 0xCC, + 0xB8, 0x05, 0x05, 0xE2, 0x8A, 0xB4, 0xCC, 0xB8, + 0x05, 0x05, 0xE2, 0x8A, 0xB5, 0xCC, 0xB8, 0x05, + 0x06, 0xCE, 0x91, 0xCC, 0x93, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0x91, 0xCC, 0x94, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0x95, 0xCC, 0x93, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0x95, 0xCC, 0x93, 0xCC, 0x81, 0xCA, + 0x06, 0xCE, 0x95, 0xCC, 0x94, 0xCC, 0x80, 0xCA, + // Bytes 3d00 - 3d3f + 0x06, 0xCE, 0x95, 0xCC, 0x94, 0xCC, 0x81, 0xCA, + 0x06, 0xCE, 0x97, 0xCC, 0x93, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0x97, 0xCC, 0x94, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0x99, 0xCC, 0x93, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0x99, 0xCC, 0x93, 0xCC, 0x81, 0xCA, + 0x06, 0xCE, 0x99, 0xCC, 0x93, 0xCD, 0x82, 0xCA, + 0x06, 0xCE, 0x99, 0xCC, 0x94, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0x99, 0xCC, 0x94, 0xCC, 0x81, 0xCA, + // Bytes 3d40 - 3d7f + 0x06, 0xCE, 0x99, 0xCC, 0x94, 0xCD, 0x82, 0xCA, + 0x06, 0xCE, 0x9F, 0xCC, 0x93, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0x9F, 0xCC, 0x93, 0xCC, 0x81, 0xCA, + 0x06, 0xCE, 0x9F, 0xCC, 0x94, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0x9F, 0xCC, 0x94, 0xCC, 0x81, 0xCA, + 0x06, 0xCE, 0xA5, 0xCC, 0x94, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0xA5, 0xCC, 0x94, 0xCC, 0x81, 0xCA, + 0x06, 0xCE, 0xA5, 0xCC, 0x94, 0xCD, 0x82, 0xCA, + // Bytes 3d80 - 3dbf + 0x06, 0xCE, 0xA9, 0xCC, 0x93, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0xA9, 0xCC, 0x94, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0xB1, 0xCC, 0x80, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0xB1, 0xCC, 0x81, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0xB1, 0xCC, 0x93, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0xB1, 0xCC, 0x94, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0xB1, 0xCD, 0x82, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0xB5, 0xCC, 0x93, 0xCC, 0x80, 0xCA, + // Bytes 3dc0 - 3dff + 0x06, 0xCE, 0xB5, 0xCC, 0x93, 0xCC, 0x81, 0xCA, + 0x06, 0xCE, 0xB5, 0xCC, 0x94, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0xB5, 0xCC, 0x94, 0xCC, 0x81, 0xCA, + 0x06, 0xCE, 0xB7, 0xCC, 0x80, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0xB7, 0xCC, 0x81, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0xB7, 0xCC, 0x93, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0xB7, 0xCC, 0x94, 0xCD, 0x85, 0xDA, + 0x06, 0xCE, 0xB7, 0xCD, 0x82, 0xCD, 0x85, 0xDA, + // Bytes 3e00 - 3e3f + 0x06, 0xCE, 0xB9, 0xCC, 0x88, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0xB9, 0xCC, 0x88, 0xCC, 0x81, 0xCA, + 0x06, 0xCE, 0xB9, 0xCC, 0x88, 0xCD, 0x82, 0xCA, + 0x06, 0xCE, 0xB9, 0xCC, 0x93, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0xB9, 0xCC, 0x93, 0xCC, 0x81, 0xCA, + 0x06, 0xCE, 0xB9, 0xCC, 0x93, 0xCD, 0x82, 0xCA, + 0x06, 0xCE, 0xB9, 0xCC, 0x94, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0xB9, 0xCC, 0x94, 0xCC, 0x81, 0xCA, + // Bytes 3e40 - 3e7f + 0x06, 0xCE, 0xB9, 0xCC, 0x94, 0xCD, 0x82, 0xCA, + 0x06, 0xCE, 0xBF, 0xCC, 0x93, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0xBF, 0xCC, 0x93, 0xCC, 0x81, 0xCA, + 0x06, 0xCE, 0xBF, 0xCC, 0x94, 0xCC, 0x80, 0xCA, + 0x06, 0xCE, 0xBF, 0xCC, 0x94, 0xCC, 0x81, 0xCA, + 0x06, 0xCF, 0x85, 0xCC, 0x88, 0xCC, 0x80, 0xCA, + 0x06, 0xCF, 0x85, 0xCC, 0x88, 0xCC, 0x81, 0xCA, + 0x06, 0xCF, 0x85, 0xCC, 0x88, 0xCD, 0x82, 0xCA, + // Bytes 3e80 - 3ebf + 0x06, 0xCF, 0x85, 0xCC, 0x93, 0xCC, 0x80, 0xCA, + 0x06, 0xCF, 0x85, 0xCC, 0x93, 0xCC, 0x81, 0xCA, + 0x06, 0xCF, 0x85, 0xCC, 0x93, 0xCD, 0x82, 0xCA, + 0x06, 0xCF, 0x85, 0xCC, 0x94, 0xCC, 0x80, 0xCA, + 0x06, 0xCF, 0x85, 0xCC, 0x94, 0xCC, 0x81, 0xCA, + 0x06, 0xCF, 0x85, 0xCC, 0x94, 0xCD, 0x82, 0xCA, + 0x06, 0xCF, 0x89, 0xCC, 0x80, 0xCD, 0x85, 0xDA, + 0x06, 0xCF, 0x89, 0xCC, 0x81, 0xCD, 0x85, 0xDA, + // Bytes 3ec0 - 3eff + 0x06, 0xCF, 0x89, 0xCC, 0x93, 0xCD, 0x85, 0xDA, + 0x06, 0xCF, 0x89, 0xCC, 0x94, 0xCD, 0x85, 0xDA, + 0x06, 0xCF, 0x89, 0xCD, 0x82, 0xCD, 0x85, 0xDA, + 0x06, 0xE0, 0xA4, 0xA8, 0xE0, 0xA4, 0xBC, 0x09, + 0x06, 0xE0, 0xA4, 0xB0, 0xE0, 0xA4, 0xBC, 0x09, + 0x06, 0xE0, 0xA4, 0xB3, 0xE0, 0xA4, 0xBC, 0x09, + 0x06, 0xE0, 0xB1, 0x86, 0xE0, 0xB1, 0x96, 0x85, + 0x06, 0xE0, 0xB7, 0x99, 0xE0, 0xB7, 0x8A, 0x11, + // Bytes 3f00 - 3f3f + 0x06, 0xE3, 0x81, 0x86, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0x8B, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0x8D, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0x8F, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0x91, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0x93, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0x95, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0x97, 0xE3, 0x82, 0x99, 0x0D, + // Bytes 3f40 - 3f7f + 0x06, 0xE3, 0x81, 0x99, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0x9B, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0x9D, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0x9F, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0xA4, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0xA6, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0xA8, 0xE3, 0x82, 0x99, 0x0D, + // Bytes 3f80 - 3fbf + 0x06, 0xE3, 0x81, 0xAF, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0xAF, 0xE3, 0x82, 0x9A, 0x0D, + 0x06, 0xE3, 0x81, 0xB2, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0xB2, 0xE3, 0x82, 0x9A, 0x0D, + 0x06, 0xE3, 0x81, 0xB5, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0xB5, 0xE3, 0x82, 0x9A, 0x0D, + 0x06, 0xE3, 0x81, 0xB8, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0xB8, 0xE3, 0x82, 0x9A, 0x0D, + // Bytes 3fc0 - 3fff + 0x06, 0xE3, 0x81, 0xBB, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x81, 0xBB, 0xE3, 0x82, 0x9A, 0x0D, + 0x06, 0xE3, 0x82, 0x9D, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x82, 0xA6, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x82, 0xAD, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x82, 0xB1, 0xE3, 0x82, 0x99, 0x0D, + // Bytes 4000 - 403f + 0x06, 0xE3, 0x82, 0xB3, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x82, 0xB5, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x82, 0xB7, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x82, 0xB9, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x82, 0xBB, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x82, 0xBD, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x82, 0xBF, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x83, 0x81, 0xE3, 0x82, 0x99, 0x0D, + // Bytes 4040 - 407f + 0x06, 0xE3, 0x83, 0x84, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x83, 0x86, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x83, 0x88, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x9A, 0x0D, + 0x06, 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x9A, 0x0D, + 0x06, 0xE3, 0x83, 0x95, 0xE3, 0x82, 0x99, 0x0D, + // Bytes 4080 - 40bf + 0x06, 0xE3, 0x83, 0x95, 0xE3, 0x82, 0x9A, 0x0D, + 0x06, 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, 0x0D, + 0x06, 0xE3, 0x83, 0x9B, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x83, 0x9B, 0xE3, 0x82, 0x9A, 0x0D, + 0x06, 0xE3, 0x83, 0xAF, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x83, 0xB0, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x83, 0xB1, 0xE3, 0x82, 0x99, 0x0D, + // Bytes 40c0 - 40ff + 0x06, 0xE3, 0x83, 0xB2, 0xE3, 0x82, 0x99, 0x0D, + 0x06, 0xE3, 0x83, 0xBD, 0xE3, 0x82, 0x99, 0x0D, + 0x08, 0xCE, 0x91, 0xCC, 0x93, 0xCC, 0x80, 0xCD, + 0x85, 0xDB, 0x08, 0xCE, 0x91, 0xCC, 0x93, 0xCC, + 0x81, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0x91, 0xCC, + 0x93, 0xCD, 0x82, 0xCD, 0x85, 0xDB, 0x08, 0xCE, + 0x91, 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, 0xDB, + 0x08, 0xCE, 0x91, 0xCC, 0x94, 0xCC, 0x81, 0xCD, + // Bytes 4100 - 413f + 0x85, 0xDB, 0x08, 0xCE, 0x91, 0xCC, 0x94, 0xCD, + 0x82, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0x97, 0xCC, + 0x93, 0xCC, 0x80, 0xCD, 0x85, 0xDB, 0x08, 0xCE, + 0x97, 0xCC, 0x93, 0xCC, 0x81, 0xCD, 0x85, 0xDB, + 0x08, 0xCE, 0x97, 0xCC, 0x93, 0xCD, 0x82, 0xCD, + 0x85, 0xDB, 0x08, 0xCE, 0x97, 0xCC, 0x94, 0xCC, + 0x80, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0x97, 0xCC, + 0x94, 0xCC, 0x81, 0xCD, 0x85, 0xDB, 0x08, 0xCE, + // Bytes 4140 - 417f + 0x97, 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, 0xDB, + 0x08, 0xCE, 0xA9, 0xCC, 0x93, 0xCC, 0x80, 0xCD, + 0x85, 0xDB, 0x08, 0xCE, 0xA9, 0xCC, 0x93, 0xCC, + 0x81, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xA9, 0xCC, + 0x93, 0xCD, 0x82, 0xCD, 0x85, 0xDB, 0x08, 0xCE, + 0xA9, 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, 0xDB, + 0x08, 0xCE, 0xA9, 0xCC, 0x94, 0xCC, 0x81, 0xCD, + 0x85, 0xDB, 0x08, 0xCE, 0xA9, 0xCC, 0x94, 0xCD, + // Bytes 4180 - 41bf + 0x82, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xB1, 0xCC, + 0x93, 0xCC, 0x80, 0xCD, 0x85, 0xDB, 0x08, 0xCE, + 0xB1, 0xCC, 0x93, 0xCC, 0x81, 0xCD, 0x85, 0xDB, + 0x08, 0xCE, 0xB1, 0xCC, 0x93, 0xCD, 0x82, 0xCD, + 0x85, 0xDB, 0x08, 0xCE, 0xB1, 0xCC, 0x94, 0xCC, + 0x80, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xB1, 0xCC, + 0x94, 0xCC, 0x81, 0xCD, 0x85, 0xDB, 0x08, 0xCE, + 0xB1, 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, 0xDB, + // Bytes 41c0 - 41ff + 0x08, 0xCE, 0xB7, 0xCC, 0x93, 0xCC, 0x80, 0xCD, + 0x85, 0xDB, 0x08, 0xCE, 0xB7, 0xCC, 0x93, 0xCC, + 0x81, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xB7, 0xCC, + 0x93, 0xCD, 0x82, 0xCD, 0x85, 0xDB, 0x08, 0xCE, + 0xB7, 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, 0xDB, + 0x08, 0xCE, 0xB7, 0xCC, 0x94, 0xCC, 0x81, 0xCD, + 0x85, 0xDB, 0x08, 0xCE, 0xB7, 0xCC, 0x94, 0xCD, + 0x82, 0xCD, 0x85, 0xDB, 0x08, 0xCF, 0x89, 0xCC, + // Bytes 4200 - 423f + 0x93, 0xCC, 0x80, 0xCD, 0x85, 0xDB, 0x08, 0xCF, + 0x89, 0xCC, 0x93, 0xCC, 0x81, 0xCD, 0x85, 0xDB, + 0x08, 0xCF, 0x89, 0xCC, 0x93, 0xCD, 0x82, 0xCD, + 0x85, 0xDB, 0x08, 0xCF, 0x89, 0xCC, 0x94, 0xCC, + 0x80, 0xCD, 0x85, 0xDB, 0x08, 0xCF, 0x89, 0xCC, + 0x94, 0xCC, 0x81, 0xCD, 0x85, 0xDB, 0x08, 0xCF, + 0x89, 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, 0xDB, + 0x08, 0xF0, 0x91, 0x82, 0x99, 0xF0, 0x91, 0x82, + // Bytes 4240 - 427f + 0xBA, 0x09, 0x08, 0xF0, 0x91, 0x82, 0x9B, 0xF0, + 0x91, 0x82, 0xBA, 0x09, 0x08, 0xF0, 0x91, 0x82, + 0xA5, 0xF0, 0x91, 0x82, 0xBA, 0x09, 0x42, 0xC2, + 0xB4, 0x01, 0x43, 0x20, 0xCC, 0x81, 0xC9, 0x43, + 0x20, 0xCC, 0x83, 0xC9, 0x43, 0x20, 0xCC, 0x84, + 0xC9, 0x43, 0x20, 0xCC, 0x85, 0xC9, 0x43, 0x20, + 0xCC, 0x86, 0xC9, 0x43, 0x20, 0xCC, 0x87, 0xC9, + 0x43, 0x20, 0xCC, 0x88, 0xC9, 0x43, 0x20, 0xCC, + // Bytes 4280 - 42bf + 0x8A, 0xC9, 0x43, 0x20, 0xCC, 0x8B, 0xC9, 0x43, + 0x20, 0xCC, 0x93, 0xC9, 0x43, 0x20, 0xCC, 0x94, + 0xC9, 0x43, 0x20, 0xCC, 0xA7, 0xA5, 0x43, 0x20, + 0xCC, 0xA8, 0xA5, 0x43, 0x20, 0xCC, 0xB3, 0xB5, + 0x43, 0x20, 0xCD, 0x82, 0xC9, 0x43, 0x20, 0xCD, + 0x85, 0xD9, 0x43, 0x20, 0xD9, 0x8B, 0x59, 0x43, + 0x20, 0xD9, 0x8C, 0x5D, 0x43, 0x20, 0xD9, 0x8D, + 0x61, 0x43, 0x20, 0xD9, 0x8E, 0x65, 0x43, 0x20, + // Bytes 42c0 - 42ff + 0xD9, 0x8F, 0x69, 0x43, 0x20, 0xD9, 0x90, 0x6D, + 0x43, 0x20, 0xD9, 0x91, 0x71, 0x43, 0x20, 0xD9, + 0x92, 0x75, 0x43, 0x41, 0xCC, 0x8A, 0xC9, 0x43, + 0x73, 0xCC, 0x87, 0xC9, 0x43, 0xE1, 0x85, 0xA1, + 0x01, 0x43, 0xE1, 0x85, 0xA2, 0x01, 0x43, 0xE1, + 0x85, 0xA3, 0x01, 0x43, 0xE1, 0x85, 0xA4, 0x01, + 0x43, 0xE1, 0x85, 0xA5, 0x01, 0x43, 0xE1, 0x85, + 0xA6, 0x01, 0x43, 0xE1, 0x85, 0xA7, 0x01, 0x43, + // Bytes 4300 - 433f + 0xE1, 0x85, 0xA8, 0x01, 0x43, 0xE1, 0x85, 0xA9, + 0x01, 0x43, 0xE1, 0x85, 0xAA, 0x01, 0x43, 0xE1, + 0x85, 0xAB, 0x01, 0x43, 0xE1, 0x85, 0xAC, 0x01, + 0x43, 0xE1, 0x85, 0xAD, 0x01, 0x43, 0xE1, 0x85, + 0xAE, 0x01, 0x43, 0xE1, 0x85, 0xAF, 0x01, 0x43, + 0xE1, 0x85, 0xB0, 0x01, 0x43, 0xE1, 0x85, 0xB1, + 0x01, 0x43, 0xE1, 0x85, 0xB2, 0x01, 0x43, 0xE1, + 0x85, 0xB3, 0x01, 0x43, 0xE1, 0x85, 0xB4, 0x01, + // Bytes 4340 - 437f + 0x43, 0xE1, 0x85, 0xB5, 0x01, 0x43, 0xE1, 0x86, + 0xAA, 0x01, 0x43, 0xE1, 0x86, 0xAC, 0x01, 0x43, + 0xE1, 0x86, 0xAD, 0x01, 0x43, 0xE1, 0x86, 0xB0, + 0x01, 0x43, 0xE1, 0x86, 0xB1, 0x01, 0x43, 0xE1, + 0x86, 0xB2, 0x01, 0x43, 0xE1, 0x86, 0xB3, 0x01, + 0x43, 0xE1, 0x86, 0xB4, 0x01, 0x43, 0xE1, 0x86, + 0xB5, 0x01, 0x44, 0x20, 0xE3, 0x82, 0x99, 0x0D, + 0x44, 0x20, 0xE3, 0x82, 0x9A, 0x0D, 0x44, 0xC2, + // Bytes 4380 - 43bf + 0xA8, 0xCC, 0x81, 0xCA, 0x44, 0xCE, 0x91, 0xCC, + 0x81, 0xC9, 0x44, 0xCE, 0x95, 0xCC, 0x81, 0xC9, + 0x44, 0xCE, 0x97, 0xCC, 0x81, 0xC9, 0x44, 0xCE, + 0x99, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0x9F, 0xCC, + 0x81, 0xC9, 0x44, 0xCE, 0xA5, 0xCC, 0x81, 0xC9, + 0x44, 0xCE, 0xA5, 0xCC, 0x88, 0xC9, 0x44, 0xCE, + 0xA9, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0xB1, 0xCC, + 0x81, 0xC9, 0x44, 0xCE, 0xB5, 0xCC, 0x81, 0xC9, + // Bytes 43c0 - 43ff + 0x44, 0xCE, 0xB7, 0xCC, 0x81, 0xC9, 0x44, 0xCE, + 0xB9, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0xBF, 0xCC, + 0x81, 0xC9, 0x44, 0xCF, 0x85, 0xCC, 0x81, 0xC9, + 0x44, 0xCF, 0x89, 0xCC, 0x81, 0xC9, 0x44, 0xD7, + 0x90, 0xD6, 0xB7, 0x31, 0x44, 0xD7, 0x90, 0xD6, + 0xB8, 0x35, 0x44, 0xD7, 0x90, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0x91, 0xD6, 0xBC, 0x41, 0x44, 0xD7, + 0x91, 0xD6, 0xBF, 0x49, 0x44, 0xD7, 0x92, 0xD6, + // Bytes 4400 - 443f + 0xBC, 0x41, 0x44, 0xD7, 0x93, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0x94, 0xD6, 0xBC, 0x41, 0x44, 0xD7, + 0x95, 0xD6, 0xB9, 0x39, 0x44, 0xD7, 0x95, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0x96, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0x98, 0xD6, 0xBC, 0x41, 0x44, 0xD7, + 0x99, 0xD6, 0xB4, 0x25, 0x44, 0xD7, 0x99, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0x9A, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0x9B, 0xD6, 0xBC, 0x41, 0x44, 0xD7, + // Bytes 4440 - 447f + 0x9B, 0xD6, 0xBF, 0x49, 0x44, 0xD7, 0x9C, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0x9E, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0xA0, 0xD6, 0xBC, 0x41, 0x44, 0xD7, + 0xA1, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xA3, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0xA4, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0xA4, 0xD6, 0xBF, 0x49, 0x44, 0xD7, + 0xA6, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xA7, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0xA8, 0xD6, 0xBC, 0x41, + // Bytes 4480 - 44bf + 0x44, 0xD7, 0xA9, 0xD6, 0xBC, 0x41, 0x44, 0xD7, + 0xA9, 0xD7, 0x81, 0x4D, 0x44, 0xD7, 0xA9, 0xD7, + 0x82, 0x51, 0x44, 0xD7, 0xAA, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0xB2, 0xD6, 0xB7, 0x31, 0x44, 0xD8, + 0xA7, 0xD9, 0x8B, 0x59, 0x44, 0xD8, 0xA7, 0xD9, + 0x93, 0xC9, 0x44, 0xD8, 0xA7, 0xD9, 0x94, 0xC9, + 0x44, 0xD8, 0xA7, 0xD9, 0x95, 0xB5, 0x44, 0xD8, + 0xB0, 0xD9, 0xB0, 0x79, 0x44, 0xD8, 0xB1, 0xD9, + // Bytes 44c0 - 44ff + 0xB0, 0x79, 0x44, 0xD9, 0x80, 0xD9, 0x8B, 0x59, + 0x44, 0xD9, 0x80, 0xD9, 0x8E, 0x65, 0x44, 0xD9, + 0x80, 0xD9, 0x8F, 0x69, 0x44, 0xD9, 0x80, 0xD9, + 0x90, 0x6D, 0x44, 0xD9, 0x80, 0xD9, 0x91, 0x71, + 0x44, 0xD9, 0x80, 0xD9, 0x92, 0x75, 0x44, 0xD9, + 0x87, 0xD9, 0xB0, 0x79, 0x44, 0xD9, 0x88, 0xD9, + 0x94, 0xC9, 0x44, 0xD9, 0x89, 0xD9, 0xB0, 0x79, + 0x44, 0xD9, 0x8A, 0xD9, 0x94, 0xC9, 0x44, 0xDB, + // Bytes 4500 - 453f + 0x92, 0xD9, 0x94, 0xC9, 0x44, 0xDB, 0x95, 0xD9, + 0x94, 0xC9, 0x45, 0x20, 0xCC, 0x88, 0xCC, 0x80, + 0xCA, 0x45, 0x20, 0xCC, 0x88, 0xCC, 0x81, 0xCA, + 0x45, 0x20, 0xCC, 0x88, 0xCD, 0x82, 0xCA, 0x45, + 0x20, 0xCC, 0x93, 0xCC, 0x80, 0xCA, 0x45, 0x20, + 0xCC, 0x93, 0xCC, 0x81, 0xCA, 0x45, 0x20, 0xCC, + 0x93, 0xCD, 0x82, 0xCA, 0x45, 0x20, 0xCC, 0x94, + 0xCC, 0x80, 0xCA, 0x45, 0x20, 0xCC, 0x94, 0xCC, + // Bytes 4540 - 457f + 0x81, 0xCA, 0x45, 0x20, 0xCC, 0x94, 0xCD, 0x82, + 0xCA, 0x45, 0x20, 0xD9, 0x8C, 0xD9, 0x91, 0x72, + 0x45, 0x20, 0xD9, 0x8D, 0xD9, 0x91, 0x72, 0x45, + 0x20, 0xD9, 0x8E, 0xD9, 0x91, 0x72, 0x45, 0x20, + 0xD9, 0x8F, 0xD9, 0x91, 0x72, 0x45, 0x20, 0xD9, + 0x90, 0xD9, 0x91, 0x72, 0x45, 0x20, 0xD9, 0x91, + 0xD9, 0xB0, 0x7A, 0x45, 0xE2, 0xAB, 0x9D, 0xCC, + 0xB8, 0x05, 0x46, 0xCE, 0xB9, 0xCC, 0x88, 0xCC, + // Bytes 4580 - 45bf + 0x81, 0xCA, 0x46, 0xCF, 0x85, 0xCC, 0x88, 0xCC, + 0x81, 0xCA, 0x46, 0xD7, 0xA9, 0xD6, 0xBC, 0xD7, + 0x81, 0x4E, 0x46, 0xD7, 0xA9, 0xD6, 0xBC, 0xD7, + 0x82, 0x52, 0x46, 0xD9, 0x80, 0xD9, 0x8E, 0xD9, + 0x91, 0x72, 0x46, 0xD9, 0x80, 0xD9, 0x8F, 0xD9, + 0x91, 0x72, 0x46, 0xD9, 0x80, 0xD9, 0x90, 0xD9, + 0x91, 0x72, 0x46, 0xE0, 0xA4, 0x95, 0xE0, 0xA4, + 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0x96, 0xE0, 0xA4, + // Bytes 45c0 - 45ff + 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0x97, 0xE0, 0xA4, + 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0x9C, 0xE0, 0xA4, + 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xA1, 0xE0, 0xA4, + 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xA2, 0xE0, 0xA4, + 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xAB, 0xE0, 0xA4, + 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xAF, 0xE0, 0xA4, + 0xBC, 0x09, 0x46, 0xE0, 0xA6, 0xA1, 0xE0, 0xA6, + 0xBC, 0x09, 0x46, 0xE0, 0xA6, 0xA2, 0xE0, 0xA6, + // Bytes 4600 - 463f + 0xBC, 0x09, 0x46, 0xE0, 0xA6, 0xAF, 0xE0, 0xA6, + 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0x96, 0xE0, 0xA8, + 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0x97, 0xE0, 0xA8, + 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0x9C, 0xE0, 0xA8, + 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0xAB, 0xE0, 0xA8, + 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0xB2, 0xE0, 0xA8, + 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0xB8, 0xE0, 0xA8, + 0xBC, 0x09, 0x46, 0xE0, 0xAC, 0xA1, 0xE0, 0xAC, + // Bytes 4640 - 467f + 0xBC, 0x09, 0x46, 0xE0, 0xAC, 0xA2, 0xE0, 0xAC, + 0xBC, 0x09, 0x46, 0xE0, 0xBE, 0xB2, 0xE0, 0xBE, + 0x80, 0x9D, 0x46, 0xE0, 0xBE, 0xB3, 0xE0, 0xBE, + 0x80, 0x9D, 0x46, 0xE3, 0x83, 0x86, 0xE3, 0x82, + 0x99, 0x0D, 0x48, 0xF0, 0x9D, 0x85, 0x97, 0xF0, + 0x9D, 0x85, 0xA5, 0xAD, 0x48, 0xF0, 0x9D, 0x85, + 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xAD, 0x48, 0xF0, + 0x9D, 0x86, 0xB9, 0xF0, 0x9D, 0x85, 0xA5, 0xAD, + // Bytes 4680 - 46bf + 0x48, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, 0x9D, 0x85, + 0xA5, 0xAD, 0x49, 0xE0, 0xBE, 0xB2, 0xE0, 0xBD, + 0xB1, 0xE0, 0xBE, 0x80, 0x9E, 0x49, 0xE0, 0xBE, + 0xB3, 0xE0, 0xBD, 0xB1, 0xE0, 0xBE, 0x80, 0x9E, + 0x4C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, + 0xA5, 0xF0, 0x9D, 0x85, 0xAE, 0xAE, 0x4C, 0xF0, + 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, + 0x9D, 0x85, 0xAF, 0xAE, 0x4C, 0xF0, 0x9D, 0x85, + // Bytes 46c0 - 46ff + 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, + 0xB0, 0xAE, 0x4C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, + 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xB1, 0xAE, + 0x4C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, + 0xA5, 0xF0, 0x9D, 0x85, 0xB2, 0xAE, 0x4C, 0xF0, + 0x9D, 0x86, 0xB9, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, + 0x9D, 0x85, 0xAE, 0xAE, 0x4C, 0xF0, 0x9D, 0x86, + 0xB9, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, + // Bytes 4700 - 473f + 0xAF, 0xAE, 0x4C, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, + 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xAE, 0xAE, + 0x4C, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, 0x9D, 0x85, + 0xA5, 0xF0, 0x9D, 0x85, 0xAF, 0xAE, 0x83, 0x41, + 0xCC, 0x82, 0xC9, 0x83, 0x41, 0xCC, 0x86, 0xC9, + 0x83, 0x41, 0xCC, 0x87, 0xC9, 0x83, 0x41, 0xCC, + 0x88, 0xC9, 0x83, 0x41, 0xCC, 0x8A, 0xC9, 0x83, + 0x41, 0xCC, 0xA3, 0xB5, 0x83, 0x43, 0xCC, 0xA7, + // Bytes 4740 - 477f + 0xA5, 0x83, 0x45, 0xCC, 0x82, 0xC9, 0x83, 0x45, + 0xCC, 0x84, 0xC9, 0x83, 0x45, 0xCC, 0xA3, 0xB5, + 0x83, 0x45, 0xCC, 0xA7, 0xA5, 0x83, 0x49, 0xCC, + 0x88, 0xC9, 0x83, 0x4C, 0xCC, 0xA3, 0xB5, 0x83, + 0x4F, 0xCC, 0x82, 0xC9, 0x83, 0x4F, 0xCC, 0x83, + 0xC9, 0x83, 0x4F, 0xCC, 0x84, 0xC9, 0x83, 0x4F, + 0xCC, 0x87, 0xC9, 0x83, 0x4F, 0xCC, 0x88, 0xC9, + 0x83, 0x4F, 0xCC, 0x9B, 0xAD, 0x83, 0x4F, 0xCC, + // Bytes 4780 - 47bf + 0xA3, 0xB5, 0x83, 0x4F, 0xCC, 0xA8, 0xA5, 0x83, + 0x52, 0xCC, 0xA3, 0xB5, 0x83, 0x53, 0xCC, 0x81, + 0xC9, 0x83, 0x53, 0xCC, 0x8C, 0xC9, 0x83, 0x53, + 0xCC, 0xA3, 0xB5, 0x83, 0x55, 0xCC, 0x83, 0xC9, + 0x83, 0x55, 0xCC, 0x84, 0xC9, 0x83, 0x55, 0xCC, + 0x88, 0xC9, 0x83, 0x55, 0xCC, 0x9B, 0xAD, 0x83, + 0x61, 0xCC, 0x82, 0xC9, 0x83, 0x61, 0xCC, 0x86, + 0xC9, 0x83, 0x61, 0xCC, 0x87, 0xC9, 0x83, 0x61, + // Bytes 47c0 - 47ff + 0xCC, 0x88, 0xC9, 0x83, 0x61, 0xCC, 0x8A, 0xC9, + 0x83, 0x61, 0xCC, 0xA3, 0xB5, 0x83, 0x63, 0xCC, + 0xA7, 0xA5, 0x83, 0x65, 0xCC, 0x82, 0xC9, 0x83, + 0x65, 0xCC, 0x84, 0xC9, 0x83, 0x65, 0xCC, 0xA3, + 0xB5, 0x83, 0x65, 0xCC, 0xA7, 0xA5, 0x83, 0x69, + 0xCC, 0x88, 0xC9, 0x83, 0x6C, 0xCC, 0xA3, 0xB5, + 0x83, 0x6F, 0xCC, 0x82, 0xC9, 0x83, 0x6F, 0xCC, + 0x83, 0xC9, 0x83, 0x6F, 0xCC, 0x84, 0xC9, 0x83, + // Bytes 4800 - 483f + 0x6F, 0xCC, 0x87, 0xC9, 0x83, 0x6F, 0xCC, 0x88, + 0xC9, 0x83, 0x6F, 0xCC, 0x9B, 0xAD, 0x83, 0x6F, + 0xCC, 0xA3, 0xB5, 0x83, 0x6F, 0xCC, 0xA8, 0xA5, + 0x83, 0x72, 0xCC, 0xA3, 0xB5, 0x83, 0x73, 0xCC, + 0x81, 0xC9, 0x83, 0x73, 0xCC, 0x8C, 0xC9, 0x83, + 0x73, 0xCC, 0xA3, 0xB5, 0x83, 0x75, 0xCC, 0x83, + 0xC9, 0x83, 0x75, 0xCC, 0x84, 0xC9, 0x83, 0x75, + 0xCC, 0x88, 0xC9, 0x83, 0x75, 0xCC, 0x9B, 0xAD, + // Bytes 4840 - 487f + 0x84, 0xCE, 0x91, 0xCC, 0x93, 0xC9, 0x84, 0xCE, + 0x91, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0x95, 0xCC, + 0x93, 0xC9, 0x84, 0xCE, 0x95, 0xCC, 0x94, 0xC9, + 0x84, 0xCE, 0x97, 0xCC, 0x93, 0xC9, 0x84, 0xCE, + 0x97, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0x99, 0xCC, + 0x93, 0xC9, 0x84, 0xCE, 0x99, 0xCC, 0x94, 0xC9, + 0x84, 0xCE, 0x9F, 0xCC, 0x93, 0xC9, 0x84, 0xCE, + 0x9F, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0xA5, 0xCC, + // Bytes 4880 - 48bf + 0x94, 0xC9, 0x84, 0xCE, 0xA9, 0xCC, 0x93, 0xC9, + 0x84, 0xCE, 0xA9, 0xCC, 0x94, 0xC9, 0x84, 0xCE, + 0xB1, 0xCC, 0x80, 0xC9, 0x84, 0xCE, 0xB1, 0xCC, + 0x81, 0xC9, 0x84, 0xCE, 0xB1, 0xCC, 0x93, 0xC9, + 0x84, 0xCE, 0xB1, 0xCC, 0x94, 0xC9, 0x84, 0xCE, + 0xB1, 0xCD, 0x82, 0xC9, 0x84, 0xCE, 0xB5, 0xCC, + 0x93, 0xC9, 0x84, 0xCE, 0xB5, 0xCC, 0x94, 0xC9, + 0x84, 0xCE, 0xB7, 0xCC, 0x80, 0xC9, 0x84, 0xCE, + // Bytes 48c0 - 48ff + 0xB7, 0xCC, 0x81, 0xC9, 0x84, 0xCE, 0xB7, 0xCC, + 0x93, 0xC9, 0x84, 0xCE, 0xB7, 0xCC, 0x94, 0xC9, + 0x84, 0xCE, 0xB7, 0xCD, 0x82, 0xC9, 0x84, 0xCE, + 0xB9, 0xCC, 0x88, 0xC9, 0x84, 0xCE, 0xB9, 0xCC, + 0x93, 0xC9, 0x84, 0xCE, 0xB9, 0xCC, 0x94, 0xC9, + 0x84, 0xCE, 0xBF, 0xCC, 0x93, 0xC9, 0x84, 0xCE, + 0xBF, 0xCC, 0x94, 0xC9, 0x84, 0xCF, 0x85, 0xCC, + 0x88, 0xC9, 0x84, 0xCF, 0x85, 0xCC, 0x93, 0xC9, + // Bytes 4900 - 493f + 0x84, 0xCF, 0x85, 0xCC, 0x94, 0xC9, 0x84, 0xCF, + 0x89, 0xCC, 0x80, 0xC9, 0x84, 0xCF, 0x89, 0xCC, + 0x81, 0xC9, 0x84, 0xCF, 0x89, 0xCC, 0x93, 0xC9, + 0x84, 0xCF, 0x89, 0xCC, 0x94, 0xC9, 0x84, 0xCF, + 0x89, 0xCD, 0x82, 0xC9, 0x86, 0xCE, 0x91, 0xCC, + 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0x91, 0xCC, + 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0x91, 0xCC, + 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0x91, 0xCC, + // Bytes 4940 - 497f + 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0x91, 0xCC, + 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0x91, 0xCC, + 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0x97, 0xCC, + 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0x97, 0xCC, + 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0x97, 0xCC, + 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0x97, 0xCC, + 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0x97, 0xCC, + 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0x97, 0xCC, + // Bytes 4980 - 49bf + 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, + 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, + 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, + 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, + 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, + 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, + 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, + 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, + // Bytes 49c0 - 49ff + 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, + 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, + 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, + 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, + 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, + 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, + 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, + 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, + // Bytes 4a00 - 4a3f + 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, + 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, + 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCF, 0x89, 0xCC, + 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCF, 0x89, 0xCC, + 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCF, 0x89, 0xCC, + 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCF, 0x89, 0xCC, + 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCF, 0x89, 0xCC, + 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCF, 0x89, 0xCC, + // Bytes 4a40 - 4a7f + 0x94, 0xCD, 0x82, 0xCA, 0x42, 0xCC, 0x80, 0xC9, + 0x32, 0x42, 0xCC, 0x81, 0xC9, 0x32, 0x42, 0xCC, + 0x93, 0xC9, 0x32, 0x44, 0xCC, 0x88, 0xCC, 0x81, + 0xCA, 0x32, 0x43, 0xE3, 0x82, 0x99, 0x0D, 0x03, + 0x43, 0xE3, 0x82, 0x9A, 0x0D, 0x03, 0x46, 0xE0, + 0xBD, 0xB1, 0xE0, 0xBD, 0xB2, 0x9E, 0x26, 0x46, + 0xE0, 0xBD, 0xB1, 0xE0, 0xBD, 0xB4, 0xA2, 0x26, + 0x46, 0xE0, 0xBD, 0xB1, 0xE0, 0xBE, 0x80, 0x9E, + // Bytes 4a80 - 4abf + 0x26, 0x00, 0x01, +} + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *nfcTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return nfcValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = nfcIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *nfcTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return nfcValues[c0] + } + i := nfcIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = nfcIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = nfcIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *nfcTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return nfcValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = nfcIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *nfcTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return nfcValues[c0] + } + i := nfcIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = nfcIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = nfcIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// nfcTrie. Total size: 10332 bytes (10.09 KiB). Checksum: ad355b768fddb1b6. +type nfcTrie struct{} + +func newNfcTrie(i int) *nfcTrie { + return &nfcTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *nfcTrie) lookupValue(n uint32, b byte) uint16 { + switch { + case n < 44: + return uint16(nfcValues[n<<6+uint32(b)]) + default: + n -= 44 + return uint16(nfcSparse.lookup(n, b)) + } +} + +// nfcValues: 46 blocks, 2944 entries, 5888 bytes +// The third block is the zero block. +var nfcValues = [2944]uint16{ + // Block 0x0, offset 0x0 + 0x3c: 0xa000, 0x3d: 0xa000, 0x3e: 0xa000, + // Block 0x1, offset 0x40 + 0x41: 0xa000, 0x42: 0xa000, 0x43: 0xa000, 0x44: 0xa000, 0x45: 0xa000, + 0x46: 0xa000, 0x47: 0xa000, 0x48: 0xa000, 0x49: 0xa000, 0x4a: 0xa000, 0x4b: 0xa000, + 0x4c: 0xa000, 0x4d: 0xa000, 0x4e: 0xa000, 0x4f: 0xa000, 0x50: 0xa000, + 0x52: 0xa000, 0x53: 0xa000, 0x54: 0xa000, 0x55: 0xa000, 0x56: 0xa000, 0x57: 0xa000, + 0x58: 0xa000, 0x59: 0xa000, 0x5a: 0xa000, + 0x61: 0xa000, 0x62: 0xa000, 0x63: 0xa000, + 0x64: 0xa000, 0x65: 0xa000, 0x66: 0xa000, 0x67: 0xa000, 0x68: 0xa000, 0x69: 0xa000, + 0x6a: 0xa000, 0x6b: 0xa000, 0x6c: 0xa000, 0x6d: 0xa000, 0x6e: 0xa000, 0x6f: 0xa000, + 0x70: 0xa000, 0x72: 0xa000, 0x73: 0xa000, 0x74: 0xa000, 0x75: 0xa000, + 0x76: 0xa000, 0x77: 0xa000, 0x78: 0xa000, 0x79: 0xa000, 0x7a: 0xa000, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc0: 0x2f6f, 0xc1: 0x2f74, 0xc2: 0x471e, 0xc3: 0x2f79, 0xc4: 0x472d, 0xc5: 0x4732, + 0xc6: 0xa000, 0xc7: 0x473c, 0xc8: 0x2fe2, 0xc9: 0x2fe7, 0xca: 0x4741, 0xcb: 0x2ffb, + 0xcc: 0x306e, 0xcd: 0x3073, 0xce: 0x3078, 0xcf: 0x4755, 0xd1: 0x3104, + 0xd2: 0x3127, 0xd3: 0x312c, 0xd4: 0x475f, 0xd5: 0x4764, 0xd6: 0x4773, + 0xd8: 0xa000, 0xd9: 0x31b3, 0xda: 0x31b8, 0xdb: 0x31bd, 0xdc: 0x47a5, 0xdd: 0x3235, + 0xe0: 0x327b, 0xe1: 0x3280, 0xe2: 0x47af, 0xe3: 0x3285, + 0xe4: 0x47be, 0xe5: 0x47c3, 0xe6: 0xa000, 0xe7: 0x47cd, 0xe8: 0x32ee, 0xe9: 0x32f3, + 0xea: 0x47d2, 0xeb: 0x3307, 0xec: 0x337f, 0xed: 0x3384, 0xee: 0x3389, 0xef: 0x47e6, + 0xf1: 0x3415, 0xf2: 0x3438, 0xf3: 0x343d, 0xf4: 0x47f0, 0xf5: 0x47f5, + 0xf6: 0x4804, 0xf8: 0xa000, 0xf9: 0x34c9, 0xfa: 0x34ce, 0xfb: 0x34d3, + 0xfc: 0x4836, 0xfd: 0x3550, 0xff: 0x3569, + // Block 0x4, offset 0x100 + 0x100: 0x2f7e, 0x101: 0x328a, 0x102: 0x4723, 0x103: 0x47b4, 0x104: 0x2f9c, 0x105: 0x32a8, + 0x106: 0x2fb0, 0x107: 0x32bc, 0x108: 0x2fb5, 0x109: 0x32c1, 0x10a: 0x2fba, 0x10b: 0x32c6, + 0x10c: 0x2fbf, 0x10d: 0x32cb, 0x10e: 0x2fc9, 0x10f: 0x32d5, + 0x112: 0x4746, 0x113: 0x47d7, 0x114: 0x2ff1, 0x115: 0x32fd, 0x116: 0x2ff6, 0x117: 0x3302, + 0x118: 0x3014, 0x119: 0x3320, 0x11a: 0x3005, 0x11b: 0x3311, 0x11c: 0x302d, 0x11d: 0x3339, + 0x11e: 0x3037, 0x11f: 0x3343, 0x120: 0x303c, 0x121: 0x3348, 0x122: 0x3046, 0x123: 0x3352, + 0x124: 0x304b, 0x125: 0x3357, 0x128: 0x307d, 0x129: 0x338e, + 0x12a: 0x3082, 0x12b: 0x3393, 0x12c: 0x3087, 0x12d: 0x3398, 0x12e: 0x30aa, 0x12f: 0x33b6, + 0x130: 0x308c, 0x134: 0x30b4, 0x135: 0x33c0, + 0x136: 0x30c8, 0x137: 0x33d9, 0x139: 0x30d2, 0x13a: 0x33e3, 0x13b: 0x30dc, + 0x13c: 0x33ed, 0x13d: 0x30d7, 0x13e: 0x33e8, + // Block 0x5, offset 0x140 + 0x143: 0x30ff, 0x144: 0x3410, 0x145: 0x3118, + 0x146: 0x3429, 0x147: 0x310e, 0x148: 0x341f, + 0x14c: 0x4769, 0x14d: 0x47fa, 0x14e: 0x3131, 0x14f: 0x3442, 0x150: 0x313b, 0x151: 0x344c, + 0x154: 0x3159, 0x155: 0x346a, 0x156: 0x3172, 0x157: 0x3483, + 0x158: 0x3163, 0x159: 0x3474, 0x15a: 0x478c, 0x15b: 0x481d, 0x15c: 0x317c, 0x15d: 0x348d, + 0x15e: 0x318b, 0x15f: 0x349c, 0x160: 0x4791, 0x161: 0x4822, 0x162: 0x31a4, 0x163: 0x34ba, + 0x164: 0x3195, 0x165: 0x34ab, 0x168: 0x479b, 0x169: 0x482c, + 0x16a: 0x47a0, 0x16b: 0x4831, 0x16c: 0x31c2, 0x16d: 0x34d8, 0x16e: 0x31cc, 0x16f: 0x34e2, + 0x170: 0x31d1, 0x171: 0x34e7, 0x172: 0x31ef, 0x173: 0x3505, 0x174: 0x3212, 0x175: 0x3528, + 0x176: 0x323a, 0x177: 0x3555, 0x178: 0x324e, 0x179: 0x325d, 0x17a: 0x357d, 0x17b: 0x3267, + 0x17c: 0x3587, 0x17d: 0x326c, 0x17e: 0x358c, 0x17f: 0xa000, + // Block 0x6, offset 0x180 + 0x184: 0x8100, 0x185: 0x8100, + 0x186: 0x8100, + 0x18d: 0x2f88, 0x18e: 0x3294, 0x18f: 0x3096, 0x190: 0x33a2, 0x191: 0x3140, + 0x192: 0x3451, 0x193: 0x31d6, 0x194: 0x34ec, 0x195: 0x39cf, 0x196: 0x3b5e, 0x197: 0x39c8, + 0x198: 0x3b57, 0x199: 0x39d6, 0x19a: 0x3b65, 0x19b: 0x39c1, 0x19c: 0x3b50, + 0x19e: 0x38b0, 0x19f: 0x3a3f, 0x1a0: 0x38a9, 0x1a1: 0x3a38, 0x1a2: 0x35b3, 0x1a3: 0x35c5, + 0x1a6: 0x3041, 0x1a7: 0x334d, 0x1a8: 0x30be, 0x1a9: 0x33cf, + 0x1aa: 0x4782, 0x1ab: 0x4813, 0x1ac: 0x3990, 0x1ad: 0x3b1f, 0x1ae: 0x35d7, 0x1af: 0x35dd, + 0x1b0: 0x33c5, 0x1b4: 0x3028, 0x1b5: 0x3334, + 0x1b8: 0x30fa, 0x1b9: 0x340b, 0x1ba: 0x38b7, 0x1bb: 0x3a46, + 0x1bc: 0x35ad, 0x1bd: 0x35bf, 0x1be: 0x35b9, 0x1bf: 0x35cb, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x2f8d, 0x1c1: 0x3299, 0x1c2: 0x2f92, 0x1c3: 0x329e, 0x1c4: 0x300a, 0x1c5: 0x3316, + 0x1c6: 0x300f, 0x1c7: 0x331b, 0x1c8: 0x309b, 0x1c9: 0x33a7, 0x1ca: 0x30a0, 0x1cb: 0x33ac, + 0x1cc: 0x3145, 0x1cd: 0x3456, 0x1ce: 0x314a, 0x1cf: 0x345b, 0x1d0: 0x3168, 0x1d1: 0x3479, + 0x1d2: 0x316d, 0x1d3: 0x347e, 0x1d4: 0x31db, 0x1d5: 0x34f1, 0x1d6: 0x31e0, 0x1d7: 0x34f6, + 0x1d8: 0x3186, 0x1d9: 0x3497, 0x1da: 0x319f, 0x1db: 0x34b5, + 0x1de: 0x305a, 0x1df: 0x3366, + 0x1e6: 0x4728, 0x1e7: 0x47b9, 0x1e8: 0x4750, 0x1e9: 0x47e1, + 0x1ea: 0x395f, 0x1eb: 0x3aee, 0x1ec: 0x393c, 0x1ed: 0x3acb, 0x1ee: 0x476e, 0x1ef: 0x47ff, + 0x1f0: 0x3958, 0x1f1: 0x3ae7, 0x1f2: 0x3244, 0x1f3: 0x355f, + // Block 0x8, offset 0x200 + 0x200: 0x9932, 0x201: 0x9932, 0x202: 0x9932, 0x203: 0x9932, 0x204: 0x9932, 0x205: 0x8132, + 0x206: 0x9932, 0x207: 0x9932, 0x208: 0x9932, 0x209: 0x9932, 0x20a: 0x9932, 0x20b: 0x9932, + 0x20c: 0x9932, 0x20d: 0x8132, 0x20e: 0x8132, 0x20f: 0x9932, 0x210: 0x8132, 0x211: 0x9932, + 0x212: 0x8132, 0x213: 0x9932, 0x214: 0x9932, 0x215: 0x8133, 0x216: 0x812d, 0x217: 0x812d, + 0x218: 0x812d, 0x219: 0x812d, 0x21a: 0x8133, 0x21b: 0x992b, 0x21c: 0x812d, 0x21d: 0x812d, + 0x21e: 0x812d, 0x21f: 0x812d, 0x220: 0x812d, 0x221: 0x8129, 0x222: 0x8129, 0x223: 0x992d, + 0x224: 0x992d, 0x225: 0x992d, 0x226: 0x992d, 0x227: 0x9929, 0x228: 0x9929, 0x229: 0x812d, + 0x22a: 0x812d, 0x22b: 0x812d, 0x22c: 0x812d, 0x22d: 0x992d, 0x22e: 0x992d, 0x22f: 0x812d, + 0x230: 0x992d, 0x231: 0x992d, 0x232: 0x812d, 0x233: 0x812d, 0x234: 0x8101, 0x235: 0x8101, + 0x236: 0x8101, 0x237: 0x8101, 0x238: 0x9901, 0x239: 0x812d, 0x23a: 0x812d, 0x23b: 0x812d, + 0x23c: 0x812d, 0x23d: 0x8132, 0x23e: 0x8132, 0x23f: 0x8132, + // Block 0x9, offset 0x240 + 0x240: 0x4a44, 0x241: 0x4a49, 0x242: 0x9932, 0x243: 0x4a4e, 0x244: 0x4a53, 0x245: 0x9936, + 0x246: 0x8132, 0x247: 0x812d, 0x248: 0x812d, 0x249: 0x812d, 0x24a: 0x8132, 0x24b: 0x8132, + 0x24c: 0x8132, 0x24d: 0x812d, 0x24e: 0x812d, 0x250: 0x8132, 0x251: 0x8132, + 0x252: 0x8132, 0x253: 0x812d, 0x254: 0x812d, 0x255: 0x812d, 0x256: 0x812d, 0x257: 0x8132, + 0x258: 0x8133, 0x259: 0x812d, 0x25a: 0x812d, 0x25b: 0x8132, 0x25c: 0x8134, 0x25d: 0x8135, + 0x25e: 0x8135, 0x25f: 0x8134, 0x260: 0x8135, 0x261: 0x8135, 0x262: 0x8134, 0x263: 0x8132, + 0x264: 0x8132, 0x265: 0x8132, 0x266: 0x8132, 0x267: 0x8132, 0x268: 0x8132, 0x269: 0x8132, + 0x26a: 0x8132, 0x26b: 0x8132, 0x26c: 0x8132, 0x26d: 0x8132, 0x26e: 0x8132, 0x26f: 0x8132, + 0x274: 0x0170, + 0x27a: 0x8100, + 0x27e: 0x0037, + // Block 0xa, offset 0x280 + 0x284: 0x8100, 0x285: 0x35a1, + 0x286: 0x35e9, 0x287: 0x00ce, 0x288: 0x3607, 0x289: 0x3613, 0x28a: 0x3625, + 0x28c: 0x3643, 0x28e: 0x3655, 0x28f: 0x3673, 0x290: 0x3e08, 0x291: 0xa000, + 0x295: 0xa000, 0x297: 0xa000, + 0x299: 0xa000, + 0x29f: 0xa000, 0x2a1: 0xa000, + 0x2a5: 0xa000, 0x2a9: 0xa000, + 0x2aa: 0x3637, 0x2ab: 0x3667, 0x2ac: 0x4894, 0x2ad: 0x3697, 0x2ae: 0x48be, 0x2af: 0x36a9, + 0x2b0: 0x3e70, 0x2b1: 0xa000, 0x2b5: 0xa000, + 0x2b7: 0xa000, 0x2b9: 0xa000, + 0x2bf: 0xa000, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x3721, 0x2c1: 0x372d, 0x2c3: 0x371b, + 0x2c6: 0xa000, 0x2c7: 0x3709, + 0x2cc: 0x375d, 0x2cd: 0x3745, 0x2ce: 0x376f, 0x2d0: 0xa000, + 0x2d3: 0xa000, 0x2d5: 0xa000, 0x2d6: 0xa000, 0x2d7: 0xa000, + 0x2d8: 0xa000, 0x2d9: 0x3751, 0x2da: 0xa000, + 0x2de: 0xa000, 0x2e3: 0xa000, + 0x2e7: 0xa000, + 0x2eb: 0xa000, 0x2ed: 0xa000, + 0x2f0: 0xa000, 0x2f3: 0xa000, 0x2f5: 0xa000, + 0x2f6: 0xa000, 0x2f7: 0xa000, 0x2f8: 0xa000, 0x2f9: 0x37d5, 0x2fa: 0xa000, + 0x2fe: 0xa000, + // Block 0xc, offset 0x300 + 0x301: 0x3733, 0x302: 0x37b7, + 0x310: 0x370f, 0x311: 0x3793, + 0x312: 0x3715, 0x313: 0x3799, 0x316: 0x3727, 0x317: 0x37ab, + 0x318: 0xa000, 0x319: 0xa000, 0x31a: 0x3829, 0x31b: 0x382f, 0x31c: 0x3739, 0x31d: 0x37bd, + 0x31e: 0x373f, 0x31f: 0x37c3, 0x322: 0x374b, 0x323: 0x37cf, + 0x324: 0x3757, 0x325: 0x37db, 0x326: 0x3763, 0x327: 0x37e7, 0x328: 0xa000, 0x329: 0xa000, + 0x32a: 0x3835, 0x32b: 0x383b, 0x32c: 0x378d, 0x32d: 0x3811, 0x32e: 0x3769, 0x32f: 0x37ed, + 0x330: 0x3775, 0x331: 0x37f9, 0x332: 0x377b, 0x333: 0x37ff, 0x334: 0x3781, 0x335: 0x3805, + 0x338: 0x3787, 0x339: 0x380b, + // Block 0xd, offset 0x340 + 0x351: 0x812d, + 0x352: 0x8132, 0x353: 0x8132, 0x354: 0x8132, 0x355: 0x8132, 0x356: 0x812d, 0x357: 0x8132, + 0x358: 0x8132, 0x359: 0x8132, 0x35a: 0x812e, 0x35b: 0x812d, 0x35c: 0x8132, 0x35d: 0x8132, + 0x35e: 0x8132, 0x35f: 0x8132, 0x360: 0x8132, 0x361: 0x8132, 0x362: 0x812d, 0x363: 0x812d, + 0x364: 0x812d, 0x365: 0x812d, 0x366: 0x812d, 0x367: 0x812d, 0x368: 0x8132, 0x369: 0x8132, + 0x36a: 0x812d, 0x36b: 0x8132, 0x36c: 0x8132, 0x36d: 0x812e, 0x36e: 0x8131, 0x36f: 0x8132, + 0x370: 0x8105, 0x371: 0x8106, 0x372: 0x8107, 0x373: 0x8108, 0x374: 0x8109, 0x375: 0x810a, + 0x376: 0x810b, 0x377: 0x810c, 0x378: 0x810d, 0x379: 0x810e, 0x37a: 0x810e, 0x37b: 0x810f, + 0x37c: 0x8110, 0x37d: 0x8111, 0x37f: 0x8112, + // Block 0xe, offset 0x380 + 0x388: 0xa000, 0x38a: 0xa000, 0x38b: 0x8116, + 0x38c: 0x8117, 0x38d: 0x8118, 0x38e: 0x8119, 0x38f: 0x811a, 0x390: 0x811b, 0x391: 0x811c, + 0x392: 0x811d, 0x393: 0x9932, 0x394: 0x9932, 0x395: 0x992d, 0x396: 0x812d, 0x397: 0x8132, + 0x398: 0x8132, 0x399: 0x8132, 0x39a: 0x8132, 0x39b: 0x8132, 0x39c: 0x812d, 0x39d: 0x8132, + 0x39e: 0x8132, 0x39f: 0x812d, + 0x3b0: 0x811e, + // Block 0xf, offset 0x3c0 + 0x3c5: 0xa000, + 0x3c6: 0x2d26, 0x3c7: 0xa000, 0x3c8: 0x2d2e, 0x3c9: 0xa000, 0x3ca: 0x2d36, 0x3cb: 0xa000, + 0x3cc: 0x2d3e, 0x3cd: 0xa000, 0x3ce: 0x2d46, 0x3d1: 0xa000, + 0x3d2: 0x2d4e, + 0x3f4: 0x8102, 0x3f5: 0x9900, + 0x3fa: 0xa000, 0x3fb: 0x2d56, + 0x3fc: 0xa000, 0x3fd: 0x2d5e, 0x3fe: 0xa000, 0x3ff: 0xa000, + // Block 0x10, offset 0x400 + 0x400: 0x2f97, 0x401: 0x32a3, 0x402: 0x2fa1, 0x403: 0x32ad, 0x404: 0x2fa6, 0x405: 0x32b2, + 0x406: 0x2fab, 0x407: 0x32b7, 0x408: 0x38cc, 0x409: 0x3a5b, 0x40a: 0x2fc4, 0x40b: 0x32d0, + 0x40c: 0x2fce, 0x40d: 0x32da, 0x40e: 0x2fdd, 0x40f: 0x32e9, 0x410: 0x2fd3, 0x411: 0x32df, + 0x412: 0x2fd8, 0x413: 0x32e4, 0x414: 0x38ef, 0x415: 0x3a7e, 0x416: 0x38f6, 0x417: 0x3a85, + 0x418: 0x3019, 0x419: 0x3325, 0x41a: 0x301e, 0x41b: 0x332a, 0x41c: 0x3904, 0x41d: 0x3a93, + 0x41e: 0x3023, 0x41f: 0x332f, 0x420: 0x3032, 0x421: 0x333e, 0x422: 0x3050, 0x423: 0x335c, + 0x424: 0x305f, 0x425: 0x336b, 0x426: 0x3055, 0x427: 0x3361, 0x428: 0x3064, 0x429: 0x3370, + 0x42a: 0x3069, 0x42b: 0x3375, 0x42c: 0x30af, 0x42d: 0x33bb, 0x42e: 0x390b, 0x42f: 0x3a9a, + 0x430: 0x30b9, 0x431: 0x33ca, 0x432: 0x30c3, 0x433: 0x33d4, 0x434: 0x30cd, 0x435: 0x33de, + 0x436: 0x475a, 0x437: 0x47eb, 0x438: 0x3912, 0x439: 0x3aa1, 0x43a: 0x30e6, 0x43b: 0x33f7, + 0x43c: 0x30e1, 0x43d: 0x33f2, 0x43e: 0x30eb, 0x43f: 0x33fc, + // Block 0x11, offset 0x440 + 0x440: 0x30f0, 0x441: 0x3401, 0x442: 0x30f5, 0x443: 0x3406, 0x444: 0x3109, 0x445: 0x341a, + 0x446: 0x3113, 0x447: 0x3424, 0x448: 0x3122, 0x449: 0x3433, 0x44a: 0x311d, 0x44b: 0x342e, + 0x44c: 0x3935, 0x44d: 0x3ac4, 0x44e: 0x3943, 0x44f: 0x3ad2, 0x450: 0x394a, 0x451: 0x3ad9, + 0x452: 0x3951, 0x453: 0x3ae0, 0x454: 0x314f, 0x455: 0x3460, 0x456: 0x3154, 0x457: 0x3465, + 0x458: 0x315e, 0x459: 0x346f, 0x45a: 0x4787, 0x45b: 0x4818, 0x45c: 0x3997, 0x45d: 0x3b26, + 0x45e: 0x3177, 0x45f: 0x3488, 0x460: 0x3181, 0x461: 0x3492, 0x462: 0x4796, 0x463: 0x4827, + 0x464: 0x399e, 0x465: 0x3b2d, 0x466: 0x39a5, 0x467: 0x3b34, 0x468: 0x39ac, 0x469: 0x3b3b, + 0x46a: 0x3190, 0x46b: 0x34a1, 0x46c: 0x319a, 0x46d: 0x34b0, 0x46e: 0x31ae, 0x46f: 0x34c4, + 0x470: 0x31a9, 0x471: 0x34bf, 0x472: 0x31ea, 0x473: 0x3500, 0x474: 0x31f9, 0x475: 0x350f, + 0x476: 0x31f4, 0x477: 0x350a, 0x478: 0x39b3, 0x479: 0x3b42, 0x47a: 0x39ba, 0x47b: 0x3b49, + 0x47c: 0x31fe, 0x47d: 0x3514, 0x47e: 0x3203, 0x47f: 0x3519, + // Block 0x12, offset 0x480 + 0x480: 0x3208, 0x481: 0x351e, 0x482: 0x320d, 0x483: 0x3523, 0x484: 0x321c, 0x485: 0x3532, + 0x486: 0x3217, 0x487: 0x352d, 0x488: 0x3221, 0x489: 0x353c, 0x48a: 0x3226, 0x48b: 0x3541, + 0x48c: 0x322b, 0x48d: 0x3546, 0x48e: 0x3249, 0x48f: 0x3564, 0x490: 0x3262, 0x491: 0x3582, + 0x492: 0x3271, 0x493: 0x3591, 0x494: 0x3276, 0x495: 0x3596, 0x496: 0x337a, 0x497: 0x34a6, + 0x498: 0x3537, 0x499: 0x3573, 0x49b: 0x35d1, + 0x4a0: 0x4737, 0x4a1: 0x47c8, 0x4a2: 0x2f83, 0x4a3: 0x328f, + 0x4a4: 0x3878, 0x4a5: 0x3a07, 0x4a6: 0x3871, 0x4a7: 0x3a00, 0x4a8: 0x3886, 0x4a9: 0x3a15, + 0x4aa: 0x387f, 0x4ab: 0x3a0e, 0x4ac: 0x38be, 0x4ad: 0x3a4d, 0x4ae: 0x3894, 0x4af: 0x3a23, + 0x4b0: 0x388d, 0x4b1: 0x3a1c, 0x4b2: 0x38a2, 0x4b3: 0x3a31, 0x4b4: 0x389b, 0x4b5: 0x3a2a, + 0x4b6: 0x38c5, 0x4b7: 0x3a54, 0x4b8: 0x474b, 0x4b9: 0x47dc, 0x4ba: 0x3000, 0x4bb: 0x330c, + 0x4bc: 0x2fec, 0x4bd: 0x32f8, 0x4be: 0x38da, 0x4bf: 0x3a69, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x38d3, 0x4c1: 0x3a62, 0x4c2: 0x38e8, 0x4c3: 0x3a77, 0x4c4: 0x38e1, 0x4c5: 0x3a70, + 0x4c6: 0x38fd, 0x4c7: 0x3a8c, 0x4c8: 0x3091, 0x4c9: 0x339d, 0x4ca: 0x30a5, 0x4cb: 0x33b1, + 0x4cc: 0x477d, 0x4cd: 0x480e, 0x4ce: 0x3136, 0x4cf: 0x3447, 0x4d0: 0x3920, 0x4d1: 0x3aaf, + 0x4d2: 0x3919, 0x4d3: 0x3aa8, 0x4d4: 0x392e, 0x4d5: 0x3abd, 0x4d6: 0x3927, 0x4d7: 0x3ab6, + 0x4d8: 0x3989, 0x4d9: 0x3b18, 0x4da: 0x396d, 0x4db: 0x3afc, 0x4dc: 0x3966, 0x4dd: 0x3af5, + 0x4de: 0x397b, 0x4df: 0x3b0a, 0x4e0: 0x3974, 0x4e1: 0x3b03, 0x4e2: 0x3982, 0x4e3: 0x3b11, + 0x4e4: 0x31e5, 0x4e5: 0x34fb, 0x4e6: 0x31c7, 0x4e7: 0x34dd, 0x4e8: 0x39e4, 0x4e9: 0x3b73, + 0x4ea: 0x39dd, 0x4eb: 0x3b6c, 0x4ec: 0x39f2, 0x4ed: 0x3b81, 0x4ee: 0x39eb, 0x4ef: 0x3b7a, + 0x4f0: 0x39f9, 0x4f1: 0x3b88, 0x4f2: 0x3230, 0x4f3: 0x354b, 0x4f4: 0x3258, 0x4f5: 0x3578, + 0x4f6: 0x3253, 0x4f7: 0x356e, 0x4f8: 0x323f, 0x4f9: 0x355a, + // Block 0x14, offset 0x500 + 0x500: 0x489a, 0x501: 0x48a0, 0x502: 0x49b4, 0x503: 0x49cc, 0x504: 0x49bc, 0x505: 0x49d4, + 0x506: 0x49c4, 0x507: 0x49dc, 0x508: 0x4840, 0x509: 0x4846, 0x50a: 0x4924, 0x50b: 0x493c, + 0x50c: 0x492c, 0x50d: 0x4944, 0x50e: 0x4934, 0x50f: 0x494c, 0x510: 0x48ac, 0x511: 0x48b2, + 0x512: 0x3db8, 0x513: 0x3dc8, 0x514: 0x3dc0, 0x515: 0x3dd0, + 0x518: 0x484c, 0x519: 0x4852, 0x51a: 0x3ce8, 0x51b: 0x3cf8, 0x51c: 0x3cf0, 0x51d: 0x3d00, + 0x520: 0x48c4, 0x521: 0x48ca, 0x522: 0x49e4, 0x523: 0x49fc, + 0x524: 0x49ec, 0x525: 0x4a04, 0x526: 0x49f4, 0x527: 0x4a0c, 0x528: 0x4858, 0x529: 0x485e, + 0x52a: 0x4954, 0x52b: 0x496c, 0x52c: 0x495c, 0x52d: 0x4974, 0x52e: 0x4964, 0x52f: 0x497c, + 0x530: 0x48dc, 0x531: 0x48e2, 0x532: 0x3e18, 0x533: 0x3e30, 0x534: 0x3e20, 0x535: 0x3e38, + 0x536: 0x3e28, 0x537: 0x3e40, 0x538: 0x4864, 0x539: 0x486a, 0x53a: 0x3d18, 0x53b: 0x3d30, + 0x53c: 0x3d20, 0x53d: 0x3d38, 0x53e: 0x3d28, 0x53f: 0x3d40, + // Block 0x15, offset 0x540 + 0x540: 0x48e8, 0x541: 0x48ee, 0x542: 0x3e48, 0x543: 0x3e58, 0x544: 0x3e50, 0x545: 0x3e60, + 0x548: 0x4870, 0x549: 0x4876, 0x54a: 0x3d48, 0x54b: 0x3d58, + 0x54c: 0x3d50, 0x54d: 0x3d60, 0x550: 0x48fa, 0x551: 0x4900, + 0x552: 0x3e80, 0x553: 0x3e98, 0x554: 0x3e88, 0x555: 0x3ea0, 0x556: 0x3e90, 0x557: 0x3ea8, + 0x559: 0x487c, 0x55b: 0x3d68, 0x55d: 0x3d70, + 0x55f: 0x3d78, 0x560: 0x4912, 0x561: 0x4918, 0x562: 0x4a14, 0x563: 0x4a2c, + 0x564: 0x4a1c, 0x565: 0x4a34, 0x566: 0x4a24, 0x567: 0x4a3c, 0x568: 0x4882, 0x569: 0x4888, + 0x56a: 0x4984, 0x56b: 0x499c, 0x56c: 0x498c, 0x56d: 0x49a4, 0x56e: 0x4994, 0x56f: 0x49ac, + 0x570: 0x488e, 0x571: 0x43b4, 0x572: 0x3691, 0x573: 0x43ba, 0x574: 0x48b8, 0x575: 0x43c0, + 0x576: 0x36a3, 0x577: 0x43c6, 0x578: 0x36c1, 0x579: 0x43cc, 0x57a: 0x36d9, 0x57b: 0x43d2, + 0x57c: 0x4906, 0x57d: 0x43d8, + // Block 0x16, offset 0x580 + 0x580: 0x3da0, 0x581: 0x3da8, 0x582: 0x4184, 0x583: 0x41a2, 0x584: 0x418e, 0x585: 0x41ac, + 0x586: 0x4198, 0x587: 0x41b6, 0x588: 0x3cd8, 0x589: 0x3ce0, 0x58a: 0x40d0, 0x58b: 0x40ee, + 0x58c: 0x40da, 0x58d: 0x40f8, 0x58e: 0x40e4, 0x58f: 0x4102, 0x590: 0x3de8, 0x591: 0x3df0, + 0x592: 0x41c0, 0x593: 0x41de, 0x594: 0x41ca, 0x595: 0x41e8, 0x596: 0x41d4, 0x597: 0x41f2, + 0x598: 0x3d08, 0x599: 0x3d10, 0x59a: 0x410c, 0x59b: 0x412a, 0x59c: 0x4116, 0x59d: 0x4134, + 0x59e: 0x4120, 0x59f: 0x413e, 0x5a0: 0x3ec0, 0x5a1: 0x3ec8, 0x5a2: 0x41fc, 0x5a3: 0x421a, + 0x5a4: 0x4206, 0x5a5: 0x4224, 0x5a6: 0x4210, 0x5a7: 0x422e, 0x5a8: 0x3d80, 0x5a9: 0x3d88, + 0x5aa: 0x4148, 0x5ab: 0x4166, 0x5ac: 0x4152, 0x5ad: 0x4170, 0x5ae: 0x415c, 0x5af: 0x417a, + 0x5b0: 0x3685, 0x5b1: 0x367f, 0x5b2: 0x3d90, 0x5b3: 0x368b, 0x5b4: 0x3d98, + 0x5b6: 0x48a6, 0x5b7: 0x3db0, 0x5b8: 0x35f5, 0x5b9: 0x35ef, 0x5ba: 0x35e3, 0x5bb: 0x4384, + 0x5bc: 0x35fb, 0x5bd: 0x8100, 0x5be: 0x01d3, 0x5bf: 0xa100, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x8100, 0x5c1: 0x35a7, 0x5c2: 0x3dd8, 0x5c3: 0x369d, 0x5c4: 0x3de0, + 0x5c6: 0x48d0, 0x5c7: 0x3df8, 0x5c8: 0x3601, 0x5c9: 0x438a, 0x5ca: 0x360d, 0x5cb: 0x4390, + 0x5cc: 0x3619, 0x5cd: 0x3b8f, 0x5ce: 0x3b96, 0x5cf: 0x3b9d, 0x5d0: 0x36b5, 0x5d1: 0x36af, + 0x5d2: 0x3e00, 0x5d3: 0x457a, 0x5d6: 0x36bb, 0x5d7: 0x3e10, + 0x5d8: 0x3631, 0x5d9: 0x362b, 0x5da: 0x361f, 0x5db: 0x4396, 0x5dd: 0x3ba4, + 0x5de: 0x3bab, 0x5df: 0x3bb2, 0x5e0: 0x36eb, 0x5e1: 0x36e5, 0x5e2: 0x3e68, 0x5e3: 0x4582, + 0x5e4: 0x36cd, 0x5e5: 0x36d3, 0x5e6: 0x36f1, 0x5e7: 0x3e78, 0x5e8: 0x3661, 0x5e9: 0x365b, + 0x5ea: 0x364f, 0x5eb: 0x43a2, 0x5ec: 0x3649, 0x5ed: 0x359b, 0x5ee: 0x437e, 0x5ef: 0x0081, + 0x5f2: 0x3eb0, 0x5f3: 0x36f7, 0x5f4: 0x3eb8, + 0x5f6: 0x491e, 0x5f7: 0x3ed0, 0x5f8: 0x363d, 0x5f9: 0x439c, 0x5fa: 0x366d, 0x5fb: 0x43ae, + 0x5fc: 0x3679, 0x5fd: 0x4256, 0x5fe: 0xa100, + // Block 0x18, offset 0x600 + 0x601: 0x3c06, 0x603: 0xa000, 0x604: 0x3c0d, 0x605: 0xa000, + 0x607: 0x3c14, 0x608: 0xa000, 0x609: 0x3c1b, + 0x60d: 0xa000, + 0x620: 0x2f65, 0x621: 0xa000, 0x622: 0x3c29, + 0x624: 0xa000, 0x625: 0xa000, + 0x62d: 0x3c22, 0x62e: 0x2f60, 0x62f: 0x2f6a, + 0x630: 0x3c30, 0x631: 0x3c37, 0x632: 0xa000, 0x633: 0xa000, 0x634: 0x3c3e, 0x635: 0x3c45, + 0x636: 0xa000, 0x637: 0xa000, 0x638: 0x3c4c, 0x639: 0x3c53, 0x63a: 0xa000, 0x63b: 0xa000, + 0x63c: 0xa000, 0x63d: 0xa000, + // Block 0x19, offset 0x640 + 0x640: 0x3c5a, 0x641: 0x3c61, 0x642: 0xa000, 0x643: 0xa000, 0x644: 0x3c76, 0x645: 0x3c7d, + 0x646: 0xa000, 0x647: 0xa000, 0x648: 0x3c84, 0x649: 0x3c8b, + 0x651: 0xa000, + 0x652: 0xa000, + 0x662: 0xa000, + 0x668: 0xa000, 0x669: 0xa000, + 0x66b: 0xa000, 0x66c: 0x3ca0, 0x66d: 0x3ca7, 0x66e: 0x3cae, 0x66f: 0x3cb5, + 0x672: 0xa000, 0x673: 0xa000, 0x674: 0xa000, 0x675: 0xa000, + // Block 0x1a, offset 0x680 + 0x686: 0xa000, 0x68b: 0xa000, + 0x68c: 0x3f08, 0x68d: 0xa000, 0x68e: 0x3f10, 0x68f: 0xa000, 0x690: 0x3f18, 0x691: 0xa000, + 0x692: 0x3f20, 0x693: 0xa000, 0x694: 0x3f28, 0x695: 0xa000, 0x696: 0x3f30, 0x697: 0xa000, + 0x698: 0x3f38, 0x699: 0xa000, 0x69a: 0x3f40, 0x69b: 0xa000, 0x69c: 0x3f48, 0x69d: 0xa000, + 0x69e: 0x3f50, 0x69f: 0xa000, 0x6a0: 0x3f58, 0x6a1: 0xa000, 0x6a2: 0x3f60, + 0x6a4: 0xa000, 0x6a5: 0x3f68, 0x6a6: 0xa000, 0x6a7: 0x3f70, 0x6a8: 0xa000, 0x6a9: 0x3f78, + 0x6af: 0xa000, + 0x6b0: 0x3f80, 0x6b1: 0x3f88, 0x6b2: 0xa000, 0x6b3: 0x3f90, 0x6b4: 0x3f98, 0x6b5: 0xa000, + 0x6b6: 0x3fa0, 0x6b7: 0x3fa8, 0x6b8: 0xa000, 0x6b9: 0x3fb0, 0x6ba: 0x3fb8, 0x6bb: 0xa000, + 0x6bc: 0x3fc0, 0x6bd: 0x3fc8, + // Block 0x1b, offset 0x6c0 + 0x6d4: 0x3f00, + 0x6d9: 0x9903, 0x6da: 0x9903, 0x6db: 0x8100, 0x6dc: 0x8100, 0x6dd: 0xa000, + 0x6de: 0x3fd0, + 0x6e6: 0xa000, + 0x6eb: 0xa000, 0x6ec: 0x3fe0, 0x6ed: 0xa000, 0x6ee: 0x3fe8, 0x6ef: 0xa000, + 0x6f0: 0x3ff0, 0x6f1: 0xa000, 0x6f2: 0x3ff8, 0x6f3: 0xa000, 0x6f4: 0x4000, 0x6f5: 0xa000, + 0x6f6: 0x4008, 0x6f7: 0xa000, 0x6f8: 0x4010, 0x6f9: 0xa000, 0x6fa: 0x4018, 0x6fb: 0xa000, + 0x6fc: 0x4020, 0x6fd: 0xa000, 0x6fe: 0x4028, 0x6ff: 0xa000, + // Block 0x1c, offset 0x700 + 0x700: 0x4030, 0x701: 0xa000, 0x702: 0x4038, 0x704: 0xa000, 0x705: 0x4040, + 0x706: 0xa000, 0x707: 0x4048, 0x708: 0xa000, 0x709: 0x4050, + 0x70f: 0xa000, 0x710: 0x4058, 0x711: 0x4060, + 0x712: 0xa000, 0x713: 0x4068, 0x714: 0x4070, 0x715: 0xa000, 0x716: 0x4078, 0x717: 0x4080, + 0x718: 0xa000, 0x719: 0x4088, 0x71a: 0x4090, 0x71b: 0xa000, 0x71c: 0x4098, 0x71d: 0x40a0, + 0x72f: 0xa000, + 0x730: 0xa000, 0x731: 0xa000, 0x732: 0xa000, 0x734: 0x3fd8, + 0x737: 0x40a8, 0x738: 0x40b0, 0x739: 0x40b8, 0x73a: 0x40c0, + 0x73d: 0xa000, 0x73e: 0x40c8, + // Block 0x1d, offset 0x740 + 0x740: 0x1377, 0x741: 0x0cfb, 0x742: 0x13d3, 0x743: 0x139f, 0x744: 0x0e57, 0x745: 0x06eb, + 0x746: 0x08df, 0x747: 0x162b, 0x748: 0x162b, 0x749: 0x0a0b, 0x74a: 0x145f, 0x74b: 0x0943, + 0x74c: 0x0a07, 0x74d: 0x0bef, 0x74e: 0x0fcf, 0x74f: 0x115f, 0x750: 0x1297, 0x751: 0x12d3, + 0x752: 0x1307, 0x753: 0x141b, 0x754: 0x0d73, 0x755: 0x0dff, 0x756: 0x0eab, 0x757: 0x0f43, + 0x758: 0x125f, 0x759: 0x1447, 0x75a: 0x1573, 0x75b: 0x070f, 0x75c: 0x08b3, 0x75d: 0x0d87, + 0x75e: 0x0ecf, 0x75f: 0x1293, 0x760: 0x15c3, 0x761: 0x0ab3, 0x762: 0x0e77, 0x763: 0x1283, + 0x764: 0x1317, 0x765: 0x0c23, 0x766: 0x11bb, 0x767: 0x12df, 0x768: 0x0b1f, 0x769: 0x0d0f, + 0x76a: 0x0e17, 0x76b: 0x0f1b, 0x76c: 0x1427, 0x76d: 0x074f, 0x76e: 0x07e7, 0x76f: 0x0853, + 0x770: 0x0c8b, 0x771: 0x0d7f, 0x772: 0x0ecb, 0x773: 0x0fef, 0x774: 0x1177, 0x775: 0x128b, + 0x776: 0x12a3, 0x777: 0x13c7, 0x778: 0x14ef, 0x779: 0x15a3, 0x77a: 0x15bf, 0x77b: 0x102b, + 0x77c: 0x106b, 0x77d: 0x1123, 0x77e: 0x1243, 0x77f: 0x147b, + // Block 0x1e, offset 0x780 + 0x780: 0x15cb, 0x781: 0x134b, 0x782: 0x09c7, 0x783: 0x0b3b, 0x784: 0x10db, 0x785: 0x119b, + 0x786: 0x0eff, 0x787: 0x1033, 0x788: 0x1397, 0x789: 0x14e7, 0x78a: 0x09c3, 0x78b: 0x0a8f, + 0x78c: 0x0d77, 0x78d: 0x0e2b, 0x78e: 0x0e5f, 0x78f: 0x1113, 0x790: 0x113b, 0x791: 0x14a7, + 0x792: 0x084f, 0x793: 0x11a7, 0x794: 0x07f3, 0x795: 0x07ef, 0x796: 0x1097, 0x797: 0x1127, + 0x798: 0x125b, 0x799: 0x14af, 0x79a: 0x1367, 0x79b: 0x0c27, 0x79c: 0x0d73, 0x79d: 0x1357, + 0x79e: 0x06f7, 0x79f: 0x0a63, 0x7a0: 0x0b93, 0x7a1: 0x0f2f, 0x7a2: 0x0faf, 0x7a3: 0x0873, + 0x7a4: 0x103b, 0x7a5: 0x075f, 0x7a6: 0x0b77, 0x7a7: 0x06d7, 0x7a8: 0x0deb, 0x7a9: 0x0ca3, + 0x7aa: 0x110f, 0x7ab: 0x08c7, 0x7ac: 0x09b3, 0x7ad: 0x0ffb, 0x7ae: 0x1263, 0x7af: 0x133b, + 0x7b0: 0x0db7, 0x7b1: 0x13f7, 0x7b2: 0x0de3, 0x7b3: 0x0c37, 0x7b4: 0x121b, 0x7b5: 0x0c57, + 0x7b6: 0x0fab, 0x7b7: 0x072b, 0x7b8: 0x07a7, 0x7b9: 0x07eb, 0x7ba: 0x0d53, 0x7bb: 0x10fb, + 0x7bc: 0x11f3, 0x7bd: 0x1347, 0x7be: 0x145b, 0x7bf: 0x085b, + // Block 0x1f, offset 0x7c0 + 0x7c0: 0x090f, 0x7c1: 0x0a17, 0x7c2: 0x0b2f, 0x7c3: 0x0cbf, 0x7c4: 0x0e7b, 0x7c5: 0x103f, + 0x7c6: 0x1497, 0x7c7: 0x157b, 0x7c8: 0x15cf, 0x7c9: 0x15e7, 0x7ca: 0x0837, 0x7cb: 0x0cf3, + 0x7cc: 0x0da3, 0x7cd: 0x13eb, 0x7ce: 0x0afb, 0x7cf: 0x0bd7, 0x7d0: 0x0bf3, 0x7d1: 0x0c83, + 0x7d2: 0x0e6b, 0x7d3: 0x0eb7, 0x7d4: 0x0f67, 0x7d5: 0x108b, 0x7d6: 0x112f, 0x7d7: 0x1193, + 0x7d8: 0x13db, 0x7d9: 0x126b, 0x7da: 0x1403, 0x7db: 0x147f, 0x7dc: 0x080f, 0x7dd: 0x083b, + 0x7de: 0x0923, 0x7df: 0x0ea7, 0x7e0: 0x12f3, 0x7e1: 0x133b, 0x7e2: 0x0b1b, 0x7e3: 0x0b8b, + 0x7e4: 0x0c4f, 0x7e5: 0x0daf, 0x7e6: 0x10d7, 0x7e7: 0x0f23, 0x7e8: 0x073b, 0x7e9: 0x097f, + 0x7ea: 0x0a63, 0x7eb: 0x0ac7, 0x7ec: 0x0b97, 0x7ed: 0x0f3f, 0x7ee: 0x0f5b, 0x7ef: 0x116b, + 0x7f0: 0x118b, 0x7f1: 0x1463, 0x7f2: 0x14e3, 0x7f3: 0x14f3, 0x7f4: 0x152f, 0x7f5: 0x0753, + 0x7f6: 0x107f, 0x7f7: 0x144f, 0x7f8: 0x14cb, 0x7f9: 0x0baf, 0x7fa: 0x0717, 0x7fb: 0x0777, + 0x7fc: 0x0a67, 0x7fd: 0x0a87, 0x7fe: 0x0caf, 0x7ff: 0x0d73, + // Block 0x20, offset 0x800 + 0x800: 0x0ec3, 0x801: 0x0fcb, 0x802: 0x1277, 0x803: 0x1417, 0x804: 0x1623, 0x805: 0x0ce3, + 0x806: 0x14a3, 0x807: 0x0833, 0x808: 0x0d2f, 0x809: 0x0d3b, 0x80a: 0x0e0f, 0x80b: 0x0e47, + 0x80c: 0x0f4b, 0x80d: 0x0fa7, 0x80e: 0x1027, 0x80f: 0x110b, 0x810: 0x153b, 0x811: 0x07af, + 0x812: 0x0c03, 0x813: 0x14b3, 0x814: 0x0767, 0x815: 0x0aab, 0x816: 0x0e2f, 0x817: 0x13df, + 0x818: 0x0b67, 0x819: 0x0bb7, 0x81a: 0x0d43, 0x81b: 0x0f2f, 0x81c: 0x14bb, 0x81d: 0x0817, + 0x81e: 0x08ff, 0x81f: 0x0a97, 0x820: 0x0cd3, 0x821: 0x0d1f, 0x822: 0x0d5f, 0x823: 0x0df3, + 0x824: 0x0f47, 0x825: 0x0fbb, 0x826: 0x1157, 0x827: 0x12f7, 0x828: 0x1303, 0x829: 0x1457, + 0x82a: 0x14d7, 0x82b: 0x0883, 0x82c: 0x0e4b, 0x82d: 0x0903, 0x82e: 0x0ec7, 0x82f: 0x0f6b, + 0x830: 0x1287, 0x831: 0x14bf, 0x832: 0x15ab, 0x833: 0x15d3, 0x834: 0x0d37, 0x835: 0x0e27, + 0x836: 0x11c3, 0x837: 0x10b7, 0x838: 0x10c3, 0x839: 0x10e7, 0x83a: 0x0f17, 0x83b: 0x0e9f, + 0x83c: 0x1363, 0x83d: 0x0733, 0x83e: 0x122b, 0x83f: 0x081b, + // Block 0x21, offset 0x840 + 0x840: 0x080b, 0x841: 0x0b0b, 0x842: 0x0c2b, 0x843: 0x10f3, 0x844: 0x0a53, 0x845: 0x0e03, + 0x846: 0x0cef, 0x847: 0x13e7, 0x848: 0x12e7, 0x849: 0x14ab, 0x84a: 0x1323, 0x84b: 0x0b27, + 0x84c: 0x0787, 0x84d: 0x095b, 0x850: 0x09af, + 0x852: 0x0cdf, 0x855: 0x07f7, 0x856: 0x0f1f, 0x857: 0x0fe3, + 0x858: 0x1047, 0x859: 0x1063, 0x85a: 0x1067, 0x85b: 0x107b, 0x85c: 0x14fb, 0x85d: 0x10eb, + 0x85e: 0x116f, 0x860: 0x128f, 0x862: 0x1353, + 0x865: 0x1407, 0x866: 0x1433, + 0x86a: 0x154f, 0x86b: 0x1553, 0x86c: 0x1557, 0x86d: 0x15bb, 0x86e: 0x142b, 0x86f: 0x14c7, + 0x870: 0x0757, 0x871: 0x077b, 0x872: 0x078f, 0x873: 0x084b, 0x874: 0x0857, 0x875: 0x0897, + 0x876: 0x094b, 0x877: 0x0967, 0x878: 0x096f, 0x879: 0x09ab, 0x87a: 0x09b7, 0x87b: 0x0a93, + 0x87c: 0x0a9b, 0x87d: 0x0ba3, 0x87e: 0x0bcb, 0x87f: 0x0bd3, + // Block 0x22, offset 0x880 + 0x880: 0x0beb, 0x881: 0x0c97, 0x882: 0x0cc7, 0x883: 0x0ce7, 0x884: 0x0d57, 0x885: 0x0e1b, + 0x886: 0x0e37, 0x887: 0x0e67, 0x888: 0x0ebb, 0x889: 0x0edb, 0x88a: 0x0f4f, 0x88b: 0x102f, + 0x88c: 0x104b, 0x88d: 0x1053, 0x88e: 0x104f, 0x88f: 0x1057, 0x890: 0x105b, 0x891: 0x105f, + 0x892: 0x1073, 0x893: 0x1077, 0x894: 0x109b, 0x895: 0x10af, 0x896: 0x10cb, 0x897: 0x112f, + 0x898: 0x1137, 0x899: 0x113f, 0x89a: 0x1153, 0x89b: 0x117b, 0x89c: 0x11cb, 0x89d: 0x11ff, + 0x89e: 0x11ff, 0x89f: 0x1267, 0x8a0: 0x130f, 0x8a1: 0x1327, 0x8a2: 0x135b, 0x8a3: 0x135f, + 0x8a4: 0x13a3, 0x8a5: 0x13a7, 0x8a6: 0x13ff, 0x8a7: 0x1407, 0x8a8: 0x14db, 0x8a9: 0x151f, + 0x8aa: 0x1537, 0x8ab: 0x0b9b, 0x8ac: 0x171e, 0x8ad: 0x11e3, + 0x8b0: 0x06df, 0x8b1: 0x07e3, 0x8b2: 0x07a3, 0x8b3: 0x074b, 0x8b4: 0x078b, 0x8b5: 0x07b7, + 0x8b6: 0x0847, 0x8b7: 0x0863, 0x8b8: 0x094b, 0x8b9: 0x0937, 0x8ba: 0x0947, 0x8bb: 0x0963, + 0x8bc: 0x09af, 0x8bd: 0x09bf, 0x8be: 0x0a03, 0x8bf: 0x0a0f, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x0a2b, 0x8c1: 0x0a3b, 0x8c2: 0x0b23, 0x8c3: 0x0b2b, 0x8c4: 0x0b5b, 0x8c5: 0x0b7b, + 0x8c6: 0x0bab, 0x8c7: 0x0bc3, 0x8c8: 0x0bb3, 0x8c9: 0x0bd3, 0x8ca: 0x0bc7, 0x8cb: 0x0beb, + 0x8cc: 0x0c07, 0x8cd: 0x0c5f, 0x8ce: 0x0c6b, 0x8cf: 0x0c73, 0x8d0: 0x0c9b, 0x8d1: 0x0cdf, + 0x8d2: 0x0d0f, 0x8d3: 0x0d13, 0x8d4: 0x0d27, 0x8d5: 0x0da7, 0x8d6: 0x0db7, 0x8d7: 0x0e0f, + 0x8d8: 0x0e5b, 0x8d9: 0x0e53, 0x8da: 0x0e67, 0x8db: 0x0e83, 0x8dc: 0x0ebb, 0x8dd: 0x1013, + 0x8de: 0x0edf, 0x8df: 0x0f13, 0x8e0: 0x0f1f, 0x8e1: 0x0f5f, 0x8e2: 0x0f7b, 0x8e3: 0x0f9f, + 0x8e4: 0x0fc3, 0x8e5: 0x0fc7, 0x8e6: 0x0fe3, 0x8e7: 0x0fe7, 0x8e8: 0x0ff7, 0x8e9: 0x100b, + 0x8ea: 0x1007, 0x8eb: 0x1037, 0x8ec: 0x10b3, 0x8ed: 0x10cb, 0x8ee: 0x10e3, 0x8ef: 0x111b, + 0x8f0: 0x112f, 0x8f1: 0x114b, 0x8f2: 0x117b, 0x8f3: 0x122f, 0x8f4: 0x1257, 0x8f5: 0x12cb, + 0x8f6: 0x1313, 0x8f7: 0x131f, 0x8f8: 0x1327, 0x8f9: 0x133f, 0x8fa: 0x1353, 0x8fb: 0x1343, + 0x8fc: 0x135b, 0x8fd: 0x1357, 0x8fe: 0x134f, 0x8ff: 0x135f, + // Block 0x24, offset 0x900 + 0x900: 0x136b, 0x901: 0x13a7, 0x902: 0x13e3, 0x903: 0x1413, 0x904: 0x144b, 0x905: 0x146b, + 0x906: 0x14b7, 0x907: 0x14db, 0x908: 0x14fb, 0x909: 0x150f, 0x90a: 0x151f, 0x90b: 0x152b, + 0x90c: 0x1537, 0x90d: 0x158b, 0x90e: 0x162b, 0x90f: 0x16b5, 0x910: 0x16b0, 0x911: 0x16e2, + 0x912: 0x0607, 0x913: 0x062f, 0x914: 0x0633, 0x915: 0x1764, 0x916: 0x1791, 0x917: 0x1809, + 0x918: 0x1617, 0x919: 0x1627, + // Block 0x25, offset 0x940 + 0x940: 0x06fb, 0x941: 0x06f3, 0x942: 0x0703, 0x943: 0x1647, 0x944: 0x0747, 0x945: 0x0757, + 0x946: 0x075b, 0x947: 0x0763, 0x948: 0x076b, 0x949: 0x076f, 0x94a: 0x077b, 0x94b: 0x0773, + 0x94c: 0x05b3, 0x94d: 0x165b, 0x94e: 0x078f, 0x94f: 0x0793, 0x950: 0x0797, 0x951: 0x07b3, + 0x952: 0x164c, 0x953: 0x05b7, 0x954: 0x079f, 0x955: 0x07bf, 0x956: 0x1656, 0x957: 0x07cf, + 0x958: 0x07d7, 0x959: 0x0737, 0x95a: 0x07df, 0x95b: 0x07e3, 0x95c: 0x1831, 0x95d: 0x07ff, + 0x95e: 0x0807, 0x95f: 0x05bf, 0x960: 0x081f, 0x961: 0x0823, 0x962: 0x082b, 0x963: 0x082f, + 0x964: 0x05c3, 0x965: 0x0847, 0x966: 0x084b, 0x967: 0x0857, 0x968: 0x0863, 0x969: 0x0867, + 0x96a: 0x086b, 0x96b: 0x0873, 0x96c: 0x0893, 0x96d: 0x0897, 0x96e: 0x089f, 0x96f: 0x08af, + 0x970: 0x08b7, 0x971: 0x08bb, 0x972: 0x08bb, 0x973: 0x08bb, 0x974: 0x166a, 0x975: 0x0e93, + 0x976: 0x08cf, 0x977: 0x08d7, 0x978: 0x166f, 0x979: 0x08e3, 0x97a: 0x08eb, 0x97b: 0x08f3, + 0x97c: 0x091b, 0x97d: 0x0907, 0x97e: 0x0913, 0x97f: 0x0917, + // Block 0x26, offset 0x980 + 0x980: 0x091f, 0x981: 0x0927, 0x982: 0x092b, 0x983: 0x0933, 0x984: 0x093b, 0x985: 0x093f, + 0x986: 0x093f, 0x987: 0x0947, 0x988: 0x094f, 0x989: 0x0953, 0x98a: 0x095f, 0x98b: 0x0983, + 0x98c: 0x0967, 0x98d: 0x0987, 0x98e: 0x096b, 0x98f: 0x0973, 0x990: 0x080b, 0x991: 0x09cf, + 0x992: 0x0997, 0x993: 0x099b, 0x994: 0x099f, 0x995: 0x0993, 0x996: 0x09a7, 0x997: 0x09a3, + 0x998: 0x09bb, 0x999: 0x1674, 0x99a: 0x09d7, 0x99b: 0x09db, 0x99c: 0x09e3, 0x99d: 0x09ef, + 0x99e: 0x09f7, 0x99f: 0x0a13, 0x9a0: 0x1679, 0x9a1: 0x167e, 0x9a2: 0x0a1f, 0x9a3: 0x0a23, + 0x9a4: 0x0a27, 0x9a5: 0x0a1b, 0x9a6: 0x0a2f, 0x9a7: 0x05c7, 0x9a8: 0x05cb, 0x9a9: 0x0a37, + 0x9aa: 0x0a3f, 0x9ab: 0x0a3f, 0x9ac: 0x1683, 0x9ad: 0x0a5b, 0x9ae: 0x0a5f, 0x9af: 0x0a63, + 0x9b0: 0x0a6b, 0x9b1: 0x1688, 0x9b2: 0x0a73, 0x9b3: 0x0a77, 0x9b4: 0x0b4f, 0x9b5: 0x0a7f, + 0x9b6: 0x05cf, 0x9b7: 0x0a8b, 0x9b8: 0x0a9b, 0x9b9: 0x0aa7, 0x9ba: 0x0aa3, 0x9bb: 0x1692, + 0x9bc: 0x0aaf, 0x9bd: 0x1697, 0x9be: 0x0abb, 0x9bf: 0x0ab7, + // Block 0x27, offset 0x9c0 + 0x9c0: 0x0abf, 0x9c1: 0x0acf, 0x9c2: 0x0ad3, 0x9c3: 0x05d3, 0x9c4: 0x0ae3, 0x9c5: 0x0aeb, + 0x9c6: 0x0aef, 0x9c7: 0x0af3, 0x9c8: 0x05d7, 0x9c9: 0x169c, 0x9ca: 0x05db, 0x9cb: 0x0b0f, + 0x9cc: 0x0b13, 0x9cd: 0x0b17, 0x9ce: 0x0b1f, 0x9cf: 0x1863, 0x9d0: 0x0b37, 0x9d1: 0x16a6, + 0x9d2: 0x16a6, 0x9d3: 0x11d7, 0x9d4: 0x0b47, 0x9d5: 0x0b47, 0x9d6: 0x05df, 0x9d7: 0x16c9, + 0x9d8: 0x179b, 0x9d9: 0x0b57, 0x9da: 0x0b5f, 0x9db: 0x05e3, 0x9dc: 0x0b73, 0x9dd: 0x0b83, + 0x9de: 0x0b87, 0x9df: 0x0b8f, 0x9e0: 0x0b9f, 0x9e1: 0x05eb, 0x9e2: 0x05e7, 0x9e3: 0x0ba3, + 0x9e4: 0x16ab, 0x9e5: 0x0ba7, 0x9e6: 0x0bbb, 0x9e7: 0x0bbf, 0x9e8: 0x0bc3, 0x9e9: 0x0bbf, + 0x9ea: 0x0bcf, 0x9eb: 0x0bd3, 0x9ec: 0x0be3, 0x9ed: 0x0bdb, 0x9ee: 0x0bdf, 0x9ef: 0x0be7, + 0x9f0: 0x0beb, 0x9f1: 0x0bef, 0x9f2: 0x0bfb, 0x9f3: 0x0bff, 0x9f4: 0x0c17, 0x9f5: 0x0c1f, + 0x9f6: 0x0c2f, 0x9f7: 0x0c43, 0x9f8: 0x16ba, 0x9f9: 0x0c3f, 0x9fa: 0x0c33, 0x9fb: 0x0c4b, + 0x9fc: 0x0c53, 0x9fd: 0x0c67, 0x9fe: 0x16bf, 0x9ff: 0x0c6f, + // Block 0x28, offset 0xa00 + 0xa00: 0x0c63, 0xa01: 0x0c5b, 0xa02: 0x05ef, 0xa03: 0x0c77, 0xa04: 0x0c7f, 0xa05: 0x0c87, + 0xa06: 0x0c7b, 0xa07: 0x05f3, 0xa08: 0x0c97, 0xa09: 0x0c9f, 0xa0a: 0x16c4, 0xa0b: 0x0ccb, + 0xa0c: 0x0cff, 0xa0d: 0x0cdb, 0xa0e: 0x05ff, 0xa0f: 0x0ce7, 0xa10: 0x05fb, 0xa11: 0x05f7, + 0xa12: 0x07c3, 0xa13: 0x07c7, 0xa14: 0x0d03, 0xa15: 0x0ceb, 0xa16: 0x11ab, 0xa17: 0x0663, + 0xa18: 0x0d0f, 0xa19: 0x0d13, 0xa1a: 0x0d17, 0xa1b: 0x0d2b, 0xa1c: 0x0d23, 0xa1d: 0x16dd, + 0xa1e: 0x0603, 0xa1f: 0x0d3f, 0xa20: 0x0d33, 0xa21: 0x0d4f, 0xa22: 0x0d57, 0xa23: 0x16e7, + 0xa24: 0x0d5b, 0xa25: 0x0d47, 0xa26: 0x0d63, 0xa27: 0x0607, 0xa28: 0x0d67, 0xa29: 0x0d6b, + 0xa2a: 0x0d6f, 0xa2b: 0x0d7b, 0xa2c: 0x16ec, 0xa2d: 0x0d83, 0xa2e: 0x060b, 0xa2f: 0x0d8f, + 0xa30: 0x16f1, 0xa31: 0x0d93, 0xa32: 0x060f, 0xa33: 0x0d9f, 0xa34: 0x0dab, 0xa35: 0x0db7, + 0xa36: 0x0dbb, 0xa37: 0x16f6, 0xa38: 0x168d, 0xa39: 0x16fb, 0xa3a: 0x0ddb, 0xa3b: 0x1700, + 0xa3c: 0x0de7, 0xa3d: 0x0def, 0xa3e: 0x0ddf, 0xa3f: 0x0dfb, + // Block 0x29, offset 0xa40 + 0xa40: 0x0e0b, 0xa41: 0x0e1b, 0xa42: 0x0e0f, 0xa43: 0x0e13, 0xa44: 0x0e1f, 0xa45: 0x0e23, + 0xa46: 0x1705, 0xa47: 0x0e07, 0xa48: 0x0e3b, 0xa49: 0x0e3f, 0xa4a: 0x0613, 0xa4b: 0x0e53, + 0xa4c: 0x0e4f, 0xa4d: 0x170a, 0xa4e: 0x0e33, 0xa4f: 0x0e6f, 0xa50: 0x170f, 0xa51: 0x1714, + 0xa52: 0x0e73, 0xa53: 0x0e87, 0xa54: 0x0e83, 0xa55: 0x0e7f, 0xa56: 0x0617, 0xa57: 0x0e8b, + 0xa58: 0x0e9b, 0xa59: 0x0e97, 0xa5a: 0x0ea3, 0xa5b: 0x1651, 0xa5c: 0x0eb3, 0xa5d: 0x1719, + 0xa5e: 0x0ebf, 0xa5f: 0x1723, 0xa60: 0x0ed3, 0xa61: 0x0edf, 0xa62: 0x0ef3, 0xa63: 0x1728, + 0xa64: 0x0f07, 0xa65: 0x0f0b, 0xa66: 0x172d, 0xa67: 0x1732, 0xa68: 0x0f27, 0xa69: 0x0f37, + 0xa6a: 0x061b, 0xa6b: 0x0f3b, 0xa6c: 0x061f, 0xa6d: 0x061f, 0xa6e: 0x0f53, 0xa6f: 0x0f57, + 0xa70: 0x0f5f, 0xa71: 0x0f63, 0xa72: 0x0f6f, 0xa73: 0x0623, 0xa74: 0x0f87, 0xa75: 0x1737, + 0xa76: 0x0fa3, 0xa77: 0x173c, 0xa78: 0x0faf, 0xa79: 0x16a1, 0xa7a: 0x0fbf, 0xa7b: 0x1741, + 0xa7c: 0x1746, 0xa7d: 0x174b, 0xa7e: 0x0627, 0xa7f: 0x062b, + // Block 0x2a, offset 0xa80 + 0xa80: 0x0ff7, 0xa81: 0x1755, 0xa82: 0x1750, 0xa83: 0x175a, 0xa84: 0x175f, 0xa85: 0x0fff, + 0xa86: 0x1003, 0xa87: 0x1003, 0xa88: 0x100b, 0xa89: 0x0633, 0xa8a: 0x100f, 0xa8b: 0x0637, + 0xa8c: 0x063b, 0xa8d: 0x1769, 0xa8e: 0x1023, 0xa8f: 0x102b, 0xa90: 0x1037, 0xa91: 0x063f, + 0xa92: 0x176e, 0xa93: 0x105b, 0xa94: 0x1773, 0xa95: 0x1778, 0xa96: 0x107b, 0xa97: 0x1093, + 0xa98: 0x0643, 0xa99: 0x109b, 0xa9a: 0x109f, 0xa9b: 0x10a3, 0xa9c: 0x177d, 0xa9d: 0x1782, + 0xa9e: 0x1782, 0xa9f: 0x10bb, 0xaa0: 0x0647, 0xaa1: 0x1787, 0xaa2: 0x10cf, 0xaa3: 0x10d3, + 0xaa4: 0x064b, 0xaa5: 0x178c, 0xaa6: 0x10ef, 0xaa7: 0x064f, 0xaa8: 0x10ff, 0xaa9: 0x10f7, + 0xaaa: 0x1107, 0xaab: 0x1796, 0xaac: 0x111f, 0xaad: 0x0653, 0xaae: 0x112b, 0xaaf: 0x1133, + 0xab0: 0x1143, 0xab1: 0x0657, 0xab2: 0x17a0, 0xab3: 0x17a5, 0xab4: 0x065b, 0xab5: 0x17aa, + 0xab6: 0x115b, 0xab7: 0x17af, 0xab8: 0x1167, 0xab9: 0x1173, 0xaba: 0x117b, 0xabb: 0x17b4, + 0xabc: 0x17b9, 0xabd: 0x118f, 0xabe: 0x17be, 0xabf: 0x1197, + // Block 0x2b, offset 0xac0 + 0xac0: 0x16ce, 0xac1: 0x065f, 0xac2: 0x11af, 0xac3: 0x11b3, 0xac4: 0x0667, 0xac5: 0x11b7, + 0xac6: 0x0a33, 0xac7: 0x17c3, 0xac8: 0x17c8, 0xac9: 0x16d3, 0xaca: 0x16d8, 0xacb: 0x11d7, + 0xacc: 0x11db, 0xacd: 0x13f3, 0xace: 0x066b, 0xacf: 0x1207, 0xad0: 0x1203, 0xad1: 0x120b, + 0xad2: 0x083f, 0xad3: 0x120f, 0xad4: 0x1213, 0xad5: 0x1217, 0xad6: 0x121f, 0xad7: 0x17cd, + 0xad8: 0x121b, 0xad9: 0x1223, 0xada: 0x1237, 0xadb: 0x123b, 0xadc: 0x1227, 0xadd: 0x123f, + 0xade: 0x1253, 0xadf: 0x1267, 0xae0: 0x1233, 0xae1: 0x1247, 0xae2: 0x124b, 0xae3: 0x124f, + 0xae4: 0x17d2, 0xae5: 0x17dc, 0xae6: 0x17d7, 0xae7: 0x066f, 0xae8: 0x126f, 0xae9: 0x1273, + 0xaea: 0x127b, 0xaeb: 0x17f0, 0xaec: 0x127f, 0xaed: 0x17e1, 0xaee: 0x0673, 0xaef: 0x0677, + 0xaf0: 0x17e6, 0xaf1: 0x17eb, 0xaf2: 0x067b, 0xaf3: 0x129f, 0xaf4: 0x12a3, 0xaf5: 0x12a7, + 0xaf6: 0x12ab, 0xaf7: 0x12b7, 0xaf8: 0x12b3, 0xaf9: 0x12bf, 0xafa: 0x12bb, 0xafb: 0x12cb, + 0xafc: 0x12c3, 0xafd: 0x12c7, 0xafe: 0x12cf, 0xaff: 0x067f, + // Block 0x2c, offset 0xb00 + 0xb00: 0x12d7, 0xb01: 0x12db, 0xb02: 0x0683, 0xb03: 0x12eb, 0xb04: 0x12ef, 0xb05: 0x17f5, + 0xb06: 0x12fb, 0xb07: 0x12ff, 0xb08: 0x0687, 0xb09: 0x130b, 0xb0a: 0x05bb, 0xb0b: 0x17fa, + 0xb0c: 0x17ff, 0xb0d: 0x068b, 0xb0e: 0x068f, 0xb0f: 0x1337, 0xb10: 0x134f, 0xb11: 0x136b, + 0xb12: 0x137b, 0xb13: 0x1804, 0xb14: 0x138f, 0xb15: 0x1393, 0xb16: 0x13ab, 0xb17: 0x13b7, + 0xb18: 0x180e, 0xb19: 0x1660, 0xb1a: 0x13c3, 0xb1b: 0x13bf, 0xb1c: 0x13cb, 0xb1d: 0x1665, + 0xb1e: 0x13d7, 0xb1f: 0x13e3, 0xb20: 0x1813, 0xb21: 0x1818, 0xb22: 0x1423, 0xb23: 0x142f, + 0xb24: 0x1437, 0xb25: 0x181d, 0xb26: 0x143b, 0xb27: 0x1467, 0xb28: 0x1473, 0xb29: 0x1477, + 0xb2a: 0x146f, 0xb2b: 0x1483, 0xb2c: 0x1487, 0xb2d: 0x1822, 0xb2e: 0x1493, 0xb2f: 0x0693, + 0xb30: 0x149b, 0xb31: 0x1827, 0xb32: 0x0697, 0xb33: 0x14d3, 0xb34: 0x0ac3, 0xb35: 0x14eb, + 0xb36: 0x182c, 0xb37: 0x1836, 0xb38: 0x069b, 0xb39: 0x069f, 0xb3a: 0x1513, 0xb3b: 0x183b, + 0xb3c: 0x06a3, 0xb3d: 0x1840, 0xb3e: 0x152b, 0xb3f: 0x152b, + // Block 0x2d, offset 0xb40 + 0xb40: 0x1533, 0xb41: 0x1845, 0xb42: 0x154b, 0xb43: 0x06a7, 0xb44: 0x155b, 0xb45: 0x1567, + 0xb46: 0x156f, 0xb47: 0x1577, 0xb48: 0x06ab, 0xb49: 0x184a, 0xb4a: 0x158b, 0xb4b: 0x15a7, + 0xb4c: 0x15b3, 0xb4d: 0x06af, 0xb4e: 0x06b3, 0xb4f: 0x15b7, 0xb50: 0x184f, 0xb51: 0x06b7, + 0xb52: 0x1854, 0xb53: 0x1859, 0xb54: 0x185e, 0xb55: 0x15db, 0xb56: 0x06bb, 0xb57: 0x15ef, + 0xb58: 0x15f7, 0xb59: 0x15fb, 0xb5a: 0x1603, 0xb5b: 0x160b, 0xb5c: 0x1613, 0xb5d: 0x1868, +} + +// nfcIndex: 22 blocks, 1408 entries, 1408 bytes +// Block 0 is the zero block. +var nfcIndex = [1408]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x2c, 0xc3: 0x01, 0xc4: 0x02, 0xc5: 0x03, 0xc6: 0x2d, 0xc7: 0x04, + 0xc8: 0x05, 0xca: 0x2e, 0xcb: 0x2f, 0xcc: 0x06, 0xcd: 0x07, 0xce: 0x08, 0xcf: 0x30, + 0xd0: 0x09, 0xd1: 0x31, 0xd2: 0x32, 0xd3: 0x0a, 0xd6: 0x0b, 0xd7: 0x33, + 0xd8: 0x34, 0xd9: 0x0c, 0xdb: 0x35, 0xdc: 0x36, 0xdd: 0x37, 0xdf: 0x38, + 0xe0: 0x02, 0xe1: 0x03, 0xe2: 0x04, 0xe3: 0x05, + 0xea: 0x06, 0xeb: 0x07, 0xec: 0x08, 0xed: 0x09, 0xef: 0x0a, + 0xf0: 0x13, + // Block 0x4, offset 0x100 + 0x120: 0x39, 0x121: 0x3a, 0x123: 0x3b, 0x124: 0x3c, 0x125: 0x3d, 0x126: 0x3e, 0x127: 0x3f, + 0x128: 0x40, 0x129: 0x41, 0x12a: 0x42, 0x12b: 0x43, 0x12c: 0x3e, 0x12d: 0x44, 0x12e: 0x45, 0x12f: 0x46, + 0x131: 0x47, 0x132: 0x48, 0x133: 0x49, 0x134: 0x4a, 0x135: 0x4b, 0x137: 0x4c, + 0x138: 0x4d, 0x139: 0x4e, 0x13a: 0x4f, 0x13b: 0x50, 0x13c: 0x51, 0x13d: 0x52, 0x13e: 0x53, 0x13f: 0x54, + // Block 0x5, offset 0x140 + 0x140: 0x55, 0x142: 0x56, 0x144: 0x57, 0x145: 0x58, 0x146: 0x59, 0x147: 0x5a, + 0x14d: 0x5b, + 0x15c: 0x5c, 0x15f: 0x5d, + 0x162: 0x5e, 0x164: 0x5f, + 0x168: 0x60, 0x169: 0x61, 0x16a: 0x62, 0x16c: 0x0d, 0x16d: 0x63, 0x16e: 0x64, 0x16f: 0x65, + 0x170: 0x66, 0x173: 0x67, 0x177: 0x68, + 0x178: 0x0e, 0x179: 0x0f, 0x17a: 0x10, 0x17b: 0x11, 0x17c: 0x12, 0x17d: 0x13, 0x17e: 0x14, 0x17f: 0x15, + // Block 0x6, offset 0x180 + 0x180: 0x69, 0x183: 0x6a, 0x184: 0x6b, 0x186: 0x6c, 0x187: 0x6d, + 0x188: 0x6e, 0x189: 0x16, 0x18a: 0x17, 0x18b: 0x6f, 0x18c: 0x70, + 0x1ab: 0x71, + 0x1b3: 0x72, 0x1b5: 0x73, 0x1b7: 0x74, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x75, 0x1c1: 0x18, 0x1c2: 0x19, 0x1c3: 0x1a, 0x1c4: 0x76, 0x1c5: 0x77, + 0x1c9: 0x78, 0x1cc: 0x79, 0x1cd: 0x7a, + // Block 0x8, offset 0x200 + 0x219: 0x7b, 0x21a: 0x7c, 0x21b: 0x7d, + 0x220: 0x7e, 0x223: 0x7f, 0x224: 0x80, 0x225: 0x81, 0x226: 0x82, 0x227: 0x83, + 0x22a: 0x84, 0x22b: 0x85, 0x22f: 0x86, + 0x230: 0x87, 0x231: 0x88, 0x232: 0x89, 0x233: 0x8a, 0x234: 0x8b, 0x235: 0x8c, 0x236: 0x8d, 0x237: 0x87, + 0x238: 0x88, 0x239: 0x89, 0x23a: 0x8a, 0x23b: 0x8b, 0x23c: 0x8c, 0x23d: 0x8d, 0x23e: 0x87, 0x23f: 0x88, + // Block 0x9, offset 0x240 + 0x240: 0x89, 0x241: 0x8a, 0x242: 0x8b, 0x243: 0x8c, 0x244: 0x8d, 0x245: 0x87, 0x246: 0x88, 0x247: 0x89, + 0x248: 0x8a, 0x249: 0x8b, 0x24a: 0x8c, 0x24b: 0x8d, 0x24c: 0x87, 0x24d: 0x88, 0x24e: 0x89, 0x24f: 0x8a, + 0x250: 0x8b, 0x251: 0x8c, 0x252: 0x8d, 0x253: 0x87, 0x254: 0x88, 0x255: 0x89, 0x256: 0x8a, 0x257: 0x8b, + 0x258: 0x8c, 0x259: 0x8d, 0x25a: 0x87, 0x25b: 0x88, 0x25c: 0x89, 0x25d: 0x8a, 0x25e: 0x8b, 0x25f: 0x8c, + 0x260: 0x8d, 0x261: 0x87, 0x262: 0x88, 0x263: 0x89, 0x264: 0x8a, 0x265: 0x8b, 0x266: 0x8c, 0x267: 0x8d, + 0x268: 0x87, 0x269: 0x88, 0x26a: 0x89, 0x26b: 0x8a, 0x26c: 0x8b, 0x26d: 0x8c, 0x26e: 0x8d, 0x26f: 0x87, + 0x270: 0x88, 0x271: 0x89, 0x272: 0x8a, 0x273: 0x8b, 0x274: 0x8c, 0x275: 0x8d, 0x276: 0x87, 0x277: 0x88, + 0x278: 0x89, 0x279: 0x8a, 0x27a: 0x8b, 0x27b: 0x8c, 0x27c: 0x8d, 0x27d: 0x87, 0x27e: 0x88, 0x27f: 0x89, + // Block 0xa, offset 0x280 + 0x280: 0x8a, 0x281: 0x8b, 0x282: 0x8c, 0x283: 0x8d, 0x284: 0x87, 0x285: 0x88, 0x286: 0x89, 0x287: 0x8a, + 0x288: 0x8b, 0x289: 0x8c, 0x28a: 0x8d, 0x28b: 0x87, 0x28c: 0x88, 0x28d: 0x89, 0x28e: 0x8a, 0x28f: 0x8b, + 0x290: 0x8c, 0x291: 0x8d, 0x292: 0x87, 0x293: 0x88, 0x294: 0x89, 0x295: 0x8a, 0x296: 0x8b, 0x297: 0x8c, + 0x298: 0x8d, 0x299: 0x87, 0x29a: 0x88, 0x29b: 0x89, 0x29c: 0x8a, 0x29d: 0x8b, 0x29e: 0x8c, 0x29f: 0x8d, + 0x2a0: 0x87, 0x2a1: 0x88, 0x2a2: 0x89, 0x2a3: 0x8a, 0x2a4: 0x8b, 0x2a5: 0x8c, 0x2a6: 0x8d, 0x2a7: 0x87, + 0x2a8: 0x88, 0x2a9: 0x89, 0x2aa: 0x8a, 0x2ab: 0x8b, 0x2ac: 0x8c, 0x2ad: 0x8d, 0x2ae: 0x87, 0x2af: 0x88, + 0x2b0: 0x89, 0x2b1: 0x8a, 0x2b2: 0x8b, 0x2b3: 0x8c, 0x2b4: 0x8d, 0x2b5: 0x87, 0x2b6: 0x88, 0x2b7: 0x89, + 0x2b8: 0x8a, 0x2b9: 0x8b, 0x2ba: 0x8c, 0x2bb: 0x8d, 0x2bc: 0x87, 0x2bd: 0x88, 0x2be: 0x89, 0x2bf: 0x8a, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x8b, 0x2c1: 0x8c, 0x2c2: 0x8d, 0x2c3: 0x87, 0x2c4: 0x88, 0x2c5: 0x89, 0x2c6: 0x8a, 0x2c7: 0x8b, + 0x2c8: 0x8c, 0x2c9: 0x8d, 0x2ca: 0x87, 0x2cb: 0x88, 0x2cc: 0x89, 0x2cd: 0x8a, 0x2ce: 0x8b, 0x2cf: 0x8c, + 0x2d0: 0x8d, 0x2d1: 0x87, 0x2d2: 0x88, 0x2d3: 0x89, 0x2d4: 0x8a, 0x2d5: 0x8b, 0x2d6: 0x8c, 0x2d7: 0x8d, + 0x2d8: 0x87, 0x2d9: 0x88, 0x2da: 0x89, 0x2db: 0x8a, 0x2dc: 0x8b, 0x2dd: 0x8c, 0x2de: 0x8e, + // Block 0xc, offset 0x300 + 0x324: 0x1b, 0x325: 0x1c, 0x326: 0x1d, 0x327: 0x1e, + 0x328: 0x1f, 0x329: 0x20, 0x32a: 0x21, 0x32b: 0x22, 0x32c: 0x8f, 0x32d: 0x90, 0x32e: 0x91, + 0x331: 0x92, 0x332: 0x93, 0x333: 0x94, 0x334: 0x95, + 0x338: 0x96, 0x339: 0x97, 0x33a: 0x98, 0x33b: 0x99, 0x33e: 0x9a, 0x33f: 0x9b, + // Block 0xd, offset 0x340 + 0x347: 0x9c, + 0x34b: 0x9d, 0x34d: 0x9e, + 0x368: 0x9f, 0x36b: 0xa0, + // Block 0xe, offset 0x380 + 0x381: 0xa1, 0x382: 0xa2, 0x384: 0xa3, 0x385: 0x82, 0x387: 0xa4, + 0x388: 0xa5, 0x38b: 0xa6, 0x38c: 0x3e, 0x38d: 0xa7, + 0x391: 0xa8, 0x392: 0xa9, 0x393: 0xaa, 0x396: 0xab, 0x397: 0xac, + 0x398: 0x73, 0x39a: 0xad, 0x39c: 0xae, + 0x3b0: 0x73, + // Block 0xf, offset 0x3c0 + 0x3eb: 0xaf, 0x3ec: 0xb0, + // Block 0x10, offset 0x400 + 0x432: 0xb1, + // Block 0x11, offset 0x440 + 0x445: 0xb2, 0x446: 0xb3, 0x447: 0xb4, + 0x449: 0xb5, + // Block 0x12, offset 0x480 + 0x480: 0xb6, + 0x4a3: 0xb7, 0x4a5: 0xb8, + // Block 0x13, offset 0x4c0 + 0x4c8: 0xb9, + // Block 0x14, offset 0x500 + 0x520: 0x23, 0x521: 0x24, 0x522: 0x25, 0x523: 0x26, 0x524: 0x27, 0x525: 0x28, 0x526: 0x29, 0x527: 0x2a, + 0x528: 0x2b, + // Block 0x15, offset 0x540 + 0x550: 0x0b, 0x551: 0x0c, 0x556: 0x0d, + 0x55b: 0x0e, 0x55d: 0x0f, 0x55e: 0x10, 0x55f: 0x11, + 0x56f: 0x12, +} + +// nfcSparseOffset: 142 entries, 284 bytes +var nfcSparseOffset = []uint16{0x0, 0x5, 0x9, 0xb, 0xd, 0x18, 0x28, 0x2a, 0x2f, 0x3a, 0x49, 0x56, 0x5e, 0x62, 0x67, 0x69, 0x7a, 0x82, 0x89, 0x8c, 0x93, 0x97, 0x9b, 0x9d, 0x9f, 0xa8, 0xac, 0xb3, 0xb8, 0xbb, 0xc5, 0xc7, 0xce, 0xd6, 0xd9, 0xdb, 0xdd, 0xdf, 0xe4, 0xf5, 0x101, 0x103, 0x109, 0x10b, 0x10d, 0x10f, 0x111, 0x113, 0x115, 0x118, 0x11b, 0x11d, 0x120, 0x123, 0x127, 0x12c, 0x135, 0x137, 0x13a, 0x13c, 0x147, 0x157, 0x15b, 0x169, 0x16c, 0x172, 0x178, 0x183, 0x187, 0x189, 0x18b, 0x18d, 0x18f, 0x191, 0x197, 0x19b, 0x19d, 0x19f, 0x1a7, 0x1ab, 0x1ae, 0x1b0, 0x1b2, 0x1b4, 0x1b7, 0x1b9, 0x1bb, 0x1bd, 0x1bf, 0x1c5, 0x1c8, 0x1ca, 0x1d1, 0x1d7, 0x1dd, 0x1e5, 0x1eb, 0x1f1, 0x1f7, 0x1fb, 0x209, 0x212, 0x215, 0x218, 0x21a, 0x21d, 0x21f, 0x223, 0x228, 0x22a, 0x22c, 0x231, 0x237, 0x239, 0x23b, 0x23d, 0x243, 0x246, 0x249, 0x251, 0x258, 0x25b, 0x25e, 0x260, 0x268, 0x26b, 0x272, 0x275, 0x27b, 0x27d, 0x280, 0x282, 0x284, 0x286, 0x288, 0x295, 0x29f, 0x2a1, 0x2a3, 0x2a9, 0x2ab, 0x2ae} + +// nfcSparseValues: 688 entries, 2752 bytes +var nfcSparseValues = [688]valueRange{ + // Block 0x0, offset 0x0 + {value: 0x0000, lo: 0x04}, + {value: 0xa100, lo: 0xa8, hi: 0xa8}, + {value: 0x8100, lo: 0xaf, hi: 0xaf}, + {value: 0x8100, lo: 0xb4, hi: 0xb4}, + {value: 0x8100, lo: 0xb8, hi: 0xb8}, + // Block 0x1, offset 0x5 + {value: 0x0091, lo: 0x03}, + {value: 0x4778, lo: 0xa0, hi: 0xa1}, + {value: 0x47aa, lo: 0xaf, hi: 0xb0}, + {value: 0xa000, lo: 0xb7, hi: 0xb7}, + // Block 0x2, offset 0x9 + {value: 0x0000, lo: 0x01}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + // Block 0x3, offset 0xb + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0x98, hi: 0x9d}, + // Block 0x4, offset 0xd + {value: 0x0006, lo: 0x0a}, + {value: 0xa000, lo: 0x81, hi: 0x81}, + {value: 0xa000, lo: 0x85, hi: 0x85}, + {value: 0xa000, lo: 0x89, hi: 0x89}, + {value: 0x48d6, lo: 0x8a, hi: 0x8a}, + {value: 0x48f4, lo: 0x8b, hi: 0x8b}, + {value: 0x36c7, lo: 0x8c, hi: 0x8c}, + {value: 0x36df, lo: 0x8d, hi: 0x8d}, + {value: 0x490c, lo: 0x8e, hi: 0x8e}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x36fd, lo: 0x93, hi: 0x94}, + // Block 0x5, offset 0x18 + {value: 0x0000, lo: 0x0f}, + {value: 0xa000, lo: 0x83, hi: 0x83}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0xa000, lo: 0x8b, hi: 0x8b}, + {value: 0xa000, lo: 0x8d, hi: 0x8d}, + {value: 0x37a5, lo: 0x90, hi: 0x90}, + {value: 0x37b1, lo: 0x91, hi: 0x91}, + {value: 0x379f, lo: 0x93, hi: 0x93}, + {value: 0xa000, lo: 0x96, hi: 0x96}, + {value: 0x3817, lo: 0x97, hi: 0x97}, + {value: 0x37e1, lo: 0x9c, hi: 0x9c}, + {value: 0x37c9, lo: 0x9d, hi: 0x9d}, + {value: 0x37f3, lo: 0x9e, hi: 0x9e}, + {value: 0xa000, lo: 0xb4, hi: 0xb5}, + {value: 0x381d, lo: 0xb6, hi: 0xb6}, + {value: 0x3823, lo: 0xb7, hi: 0xb7}, + // Block 0x6, offset 0x28 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x83, hi: 0x87}, + // Block 0x7, offset 0x2a + {value: 0x0001, lo: 0x04}, + {value: 0x8113, lo: 0x81, hi: 0x82}, + {value: 0x8132, lo: 0x84, hi: 0x84}, + {value: 0x812d, lo: 0x85, hi: 0x85}, + {value: 0x810d, lo: 0x87, hi: 0x87}, + // Block 0x8, offset 0x2f + {value: 0x0000, lo: 0x0a}, + {value: 0x8132, lo: 0x90, hi: 0x97}, + {value: 0x8119, lo: 0x98, hi: 0x98}, + {value: 0x811a, lo: 0x99, hi: 0x99}, + {value: 0x811b, lo: 0x9a, hi: 0x9a}, + {value: 0x3841, lo: 0xa2, hi: 0xa2}, + {value: 0x3847, lo: 0xa3, hi: 0xa3}, + {value: 0x3853, lo: 0xa4, hi: 0xa4}, + {value: 0x384d, lo: 0xa5, hi: 0xa5}, + {value: 0x3859, lo: 0xa6, hi: 0xa6}, + {value: 0xa000, lo: 0xa7, hi: 0xa7}, + // Block 0x9, offset 0x3a + {value: 0x0000, lo: 0x0e}, + {value: 0x386b, lo: 0x80, hi: 0x80}, + {value: 0xa000, lo: 0x81, hi: 0x81}, + {value: 0x385f, lo: 0x82, hi: 0x82}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x3865, lo: 0x93, hi: 0x93}, + {value: 0xa000, lo: 0x95, hi: 0x95}, + {value: 0x8132, lo: 0x96, hi: 0x9c}, + {value: 0x8132, lo: 0x9f, hi: 0xa2}, + {value: 0x812d, lo: 0xa3, hi: 0xa3}, + {value: 0x8132, lo: 0xa4, hi: 0xa4}, + {value: 0x8132, lo: 0xa7, hi: 0xa8}, + {value: 0x812d, lo: 0xaa, hi: 0xaa}, + {value: 0x8132, lo: 0xab, hi: 0xac}, + {value: 0x812d, lo: 0xad, hi: 0xad}, + // Block 0xa, offset 0x49 + {value: 0x0000, lo: 0x0c}, + {value: 0x811f, lo: 0x91, hi: 0x91}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + {value: 0x812d, lo: 0xb1, hi: 0xb1}, + {value: 0x8132, lo: 0xb2, hi: 0xb3}, + {value: 0x812d, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb5, hi: 0xb6}, + {value: 0x812d, lo: 0xb7, hi: 0xb9}, + {value: 0x8132, lo: 0xba, hi: 0xba}, + {value: 0x812d, lo: 0xbb, hi: 0xbc}, + {value: 0x8132, lo: 0xbd, hi: 0xbd}, + {value: 0x812d, lo: 0xbe, hi: 0xbe}, + {value: 0x8132, lo: 0xbf, hi: 0xbf}, + // Block 0xb, offset 0x56 + {value: 0x0005, lo: 0x07}, + {value: 0x8132, lo: 0x80, hi: 0x80}, + {value: 0x8132, lo: 0x81, hi: 0x81}, + {value: 0x812d, lo: 0x82, hi: 0x83}, + {value: 0x812d, lo: 0x84, hi: 0x85}, + {value: 0x812d, lo: 0x86, hi: 0x87}, + {value: 0x812d, lo: 0x88, hi: 0x89}, + {value: 0x8132, lo: 0x8a, hi: 0x8a}, + // Block 0xc, offset 0x5e + {value: 0x0000, lo: 0x03}, + {value: 0x8132, lo: 0xab, hi: 0xb1}, + {value: 0x812d, lo: 0xb2, hi: 0xb2}, + {value: 0x8132, lo: 0xb3, hi: 0xb3}, + // Block 0xd, offset 0x62 + {value: 0x0000, lo: 0x04}, + {value: 0x8132, lo: 0x96, hi: 0x99}, + {value: 0x8132, lo: 0x9b, hi: 0xa3}, + {value: 0x8132, lo: 0xa5, hi: 0xa7}, + {value: 0x8132, lo: 0xa9, hi: 0xad}, + // Block 0xe, offset 0x67 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x99, hi: 0x9b}, + // Block 0xf, offset 0x69 + {value: 0x0000, lo: 0x10}, + {value: 0x8132, lo: 0x94, hi: 0xa1}, + {value: 0x812d, lo: 0xa3, hi: 0xa3}, + {value: 0x8132, lo: 0xa4, hi: 0xa5}, + {value: 0x812d, lo: 0xa6, hi: 0xa6}, + {value: 0x8132, lo: 0xa7, hi: 0xa8}, + {value: 0x812d, lo: 0xa9, hi: 0xa9}, + {value: 0x8132, lo: 0xaa, hi: 0xac}, + {value: 0x812d, lo: 0xad, hi: 0xaf}, + {value: 0x8116, lo: 0xb0, hi: 0xb0}, + {value: 0x8117, lo: 0xb1, hi: 0xb1}, + {value: 0x8118, lo: 0xb2, hi: 0xb2}, + {value: 0x8132, lo: 0xb3, hi: 0xb5}, + {value: 0x812d, lo: 0xb6, hi: 0xb6}, + {value: 0x8132, lo: 0xb7, hi: 0xb8}, + {value: 0x812d, lo: 0xb9, hi: 0xba}, + {value: 0x8132, lo: 0xbb, hi: 0xbf}, + // Block 0x10, offset 0x7a + {value: 0x0000, lo: 0x07}, + {value: 0xa000, lo: 0xa8, hi: 0xa8}, + {value: 0x3ed8, lo: 0xa9, hi: 0xa9}, + {value: 0xa000, lo: 0xb0, hi: 0xb0}, + {value: 0x3ee0, lo: 0xb1, hi: 0xb1}, + {value: 0xa000, lo: 0xb3, hi: 0xb3}, + {value: 0x3ee8, lo: 0xb4, hi: 0xb4}, + {value: 0x9902, lo: 0xbc, hi: 0xbc}, + // Block 0x11, offset 0x82 + {value: 0x0008, lo: 0x06}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x8132, lo: 0x91, hi: 0x91}, + {value: 0x812d, lo: 0x92, hi: 0x92}, + {value: 0x8132, lo: 0x93, hi: 0x93}, + {value: 0x8132, lo: 0x94, hi: 0x94}, + {value: 0x45b2, lo: 0x98, hi: 0x9f}, + // Block 0x12, offset 0x89 + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x13, offset 0x8c + {value: 0x0008, lo: 0x06}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2c9e, lo: 0x8b, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + {value: 0x45f2, lo: 0x9c, hi: 0x9d}, + {value: 0x4602, lo: 0x9f, hi: 0x9f}, + // Block 0x14, offset 0x93 + {value: 0x0000, lo: 0x03}, + {value: 0x462a, lo: 0xb3, hi: 0xb3}, + {value: 0x4632, lo: 0xb6, hi: 0xb6}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + // Block 0x15, offset 0x97 + {value: 0x0008, lo: 0x03}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x460a, lo: 0x99, hi: 0x9b}, + {value: 0x4622, lo: 0x9e, hi: 0x9e}, + // Block 0x16, offset 0x9b + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + // Block 0x17, offset 0x9d + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + // Block 0x18, offset 0x9f + {value: 0x0000, lo: 0x08}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2cb6, lo: 0x88, hi: 0x88}, + {value: 0x2cae, lo: 0x8b, hi: 0x8b}, + {value: 0x2cbe, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x96, hi: 0x97}, + {value: 0x463a, lo: 0x9c, hi: 0x9c}, + {value: 0x4642, lo: 0x9d, hi: 0x9d}, + // Block 0x19, offset 0xa8 + {value: 0x0000, lo: 0x03}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x2cc6, lo: 0x94, hi: 0x94}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x1a, offset 0xac + {value: 0x0000, lo: 0x06}, + {value: 0xa000, lo: 0x86, hi: 0x87}, + {value: 0x2cce, lo: 0x8a, hi: 0x8a}, + {value: 0x2cde, lo: 0x8b, hi: 0x8b}, + {value: 0x2cd6, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + // Block 0x1b, offset 0xb3 + {value: 0x1801, lo: 0x04}, + {value: 0xa000, lo: 0x86, hi: 0x86}, + {value: 0x3ef0, lo: 0x88, hi: 0x88}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x8120, lo: 0x95, hi: 0x96}, + // Block 0x1c, offset 0xb8 + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + {value: 0xa000, lo: 0xbf, hi: 0xbf}, + // Block 0x1d, offset 0xbb + {value: 0x0000, lo: 0x09}, + {value: 0x2ce6, lo: 0x80, hi: 0x80}, + {value: 0x9900, lo: 0x82, hi: 0x82}, + {value: 0xa000, lo: 0x86, hi: 0x86}, + {value: 0x2cee, lo: 0x87, hi: 0x87}, + {value: 0x2cf6, lo: 0x88, hi: 0x88}, + {value: 0x2f50, lo: 0x8a, hi: 0x8a}, + {value: 0x2dd8, lo: 0x8b, hi: 0x8b}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x95, hi: 0x96}, + // Block 0x1e, offset 0xc5 + {value: 0x0000, lo: 0x01}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x1f, offset 0xc7 + {value: 0x0000, lo: 0x06}, + {value: 0xa000, lo: 0x86, hi: 0x87}, + {value: 0x2cfe, lo: 0x8a, hi: 0x8a}, + {value: 0x2d0e, lo: 0x8b, hi: 0x8b}, + {value: 0x2d06, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + // Block 0x20, offset 0xce + {value: 0x6bea, lo: 0x07}, + {value: 0x9904, lo: 0x8a, hi: 0x8a}, + {value: 0x9900, lo: 0x8f, hi: 0x8f}, + {value: 0xa000, lo: 0x99, hi: 0x99}, + {value: 0x3ef8, lo: 0x9a, hi: 0x9a}, + {value: 0x2f58, lo: 0x9c, hi: 0x9c}, + {value: 0x2de3, lo: 0x9d, hi: 0x9d}, + {value: 0x2d16, lo: 0x9e, hi: 0x9f}, + // Block 0x21, offset 0xd6 + {value: 0x0000, lo: 0x02}, + {value: 0x8122, lo: 0xb8, hi: 0xb9}, + {value: 0x8104, lo: 0xba, hi: 0xba}, + // Block 0x22, offset 0xd9 + {value: 0x0000, lo: 0x01}, + {value: 0x8123, lo: 0x88, hi: 0x8b}, + // Block 0x23, offset 0xdb + {value: 0x0000, lo: 0x01}, + {value: 0x8124, lo: 0xb8, hi: 0xb9}, + // Block 0x24, offset 0xdd + {value: 0x0000, lo: 0x01}, + {value: 0x8125, lo: 0x88, hi: 0x8b}, + // Block 0x25, offset 0xdf + {value: 0x0000, lo: 0x04}, + {value: 0x812d, lo: 0x98, hi: 0x99}, + {value: 0x812d, lo: 0xb5, hi: 0xb5}, + {value: 0x812d, lo: 0xb7, hi: 0xb7}, + {value: 0x812b, lo: 0xb9, hi: 0xb9}, + // Block 0x26, offset 0xe4 + {value: 0x0000, lo: 0x10}, + {value: 0x2644, lo: 0x83, hi: 0x83}, + {value: 0x264b, lo: 0x8d, hi: 0x8d}, + {value: 0x2652, lo: 0x92, hi: 0x92}, + {value: 0x2659, lo: 0x97, hi: 0x97}, + {value: 0x2660, lo: 0x9c, hi: 0x9c}, + {value: 0x263d, lo: 0xa9, hi: 0xa9}, + {value: 0x8126, lo: 0xb1, hi: 0xb1}, + {value: 0x8127, lo: 0xb2, hi: 0xb2}, + {value: 0x4a66, lo: 0xb3, hi: 0xb3}, + {value: 0x8128, lo: 0xb4, hi: 0xb4}, + {value: 0x4a6f, lo: 0xb5, hi: 0xb5}, + {value: 0x464a, lo: 0xb6, hi: 0xb6}, + {value: 0x8200, lo: 0xb7, hi: 0xb7}, + {value: 0x4652, lo: 0xb8, hi: 0xb8}, + {value: 0x8200, lo: 0xb9, hi: 0xb9}, + {value: 0x8127, lo: 0xba, hi: 0xbd}, + // Block 0x27, offset 0xf5 + {value: 0x0000, lo: 0x0b}, + {value: 0x8127, lo: 0x80, hi: 0x80}, + {value: 0x4a78, lo: 0x81, hi: 0x81}, + {value: 0x8132, lo: 0x82, hi: 0x83}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0x86, hi: 0x87}, + {value: 0x266e, lo: 0x93, hi: 0x93}, + {value: 0x2675, lo: 0x9d, hi: 0x9d}, + {value: 0x267c, lo: 0xa2, hi: 0xa2}, + {value: 0x2683, lo: 0xa7, hi: 0xa7}, + {value: 0x268a, lo: 0xac, hi: 0xac}, + {value: 0x2667, lo: 0xb9, hi: 0xb9}, + // Block 0x28, offset 0x101 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x86, hi: 0x86}, + // Block 0x29, offset 0x103 + {value: 0x0000, lo: 0x05}, + {value: 0xa000, lo: 0xa5, hi: 0xa5}, + {value: 0x2d1e, lo: 0xa6, hi: 0xa6}, + {value: 0x9900, lo: 0xae, hi: 0xae}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + {value: 0x8104, lo: 0xb9, hi: 0xba}, + // Block 0x2a, offset 0x109 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x8d, hi: 0x8d}, + // Block 0x2b, offset 0x10b + {value: 0x0000, lo: 0x01}, + {value: 0xa000, lo: 0x80, hi: 0x92}, + // Block 0x2c, offset 0x10d + {value: 0x0000, lo: 0x01}, + {value: 0xb900, lo: 0xa1, hi: 0xb5}, + // Block 0x2d, offset 0x10f + {value: 0x0000, lo: 0x01}, + {value: 0x9900, lo: 0xa8, hi: 0xbf}, + // Block 0x2e, offset 0x111 + {value: 0x0000, lo: 0x01}, + {value: 0x9900, lo: 0x80, hi: 0x82}, + // Block 0x2f, offset 0x113 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x9d, hi: 0x9f}, + // Block 0x30, offset 0x115 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x94, hi: 0x94}, + {value: 0x8104, lo: 0xb4, hi: 0xb4}, + // Block 0x31, offset 0x118 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x92, hi: 0x92}, + {value: 0x8132, lo: 0x9d, hi: 0x9d}, + // Block 0x32, offset 0x11b + {value: 0x0000, lo: 0x01}, + {value: 0x8131, lo: 0xa9, hi: 0xa9}, + // Block 0x33, offset 0x11d + {value: 0x0004, lo: 0x02}, + {value: 0x812e, lo: 0xb9, hi: 0xba}, + {value: 0x812d, lo: 0xbb, hi: 0xbb}, + // Block 0x34, offset 0x120 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x97, hi: 0x97}, + {value: 0x812d, lo: 0x98, hi: 0x98}, + // Block 0x35, offset 0x123 + {value: 0x0000, lo: 0x03}, + {value: 0x8104, lo: 0xa0, hi: 0xa0}, + {value: 0x8132, lo: 0xb5, hi: 0xbc}, + {value: 0x812d, lo: 0xbf, hi: 0xbf}, + // Block 0x36, offset 0x127 + {value: 0x0000, lo: 0x04}, + {value: 0x8132, lo: 0xb0, hi: 0xb4}, + {value: 0x812d, lo: 0xb5, hi: 0xba}, + {value: 0x8132, lo: 0xbb, hi: 0xbc}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + // Block 0x37, offset 0x12c + {value: 0x0000, lo: 0x08}, + {value: 0x2d66, lo: 0x80, hi: 0x80}, + {value: 0x2d6e, lo: 0x81, hi: 0x81}, + {value: 0xa000, lo: 0x82, hi: 0x82}, + {value: 0x2d76, lo: 0x83, hi: 0x83}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0xab, hi: 0xab}, + {value: 0x812d, lo: 0xac, hi: 0xac}, + {value: 0x8132, lo: 0xad, hi: 0xb3}, + // Block 0x38, offset 0x135 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xaa, hi: 0xab}, + // Block 0x39, offset 0x137 + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xa6, hi: 0xa6}, + {value: 0x8104, lo: 0xb2, hi: 0xb3}, + // Block 0x3a, offset 0x13a + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + // Block 0x3b, offset 0x13c + {value: 0x0000, lo: 0x0a}, + {value: 0x8132, lo: 0x90, hi: 0x92}, + {value: 0x8101, lo: 0x94, hi: 0x94}, + {value: 0x812d, lo: 0x95, hi: 0x99}, + {value: 0x8132, lo: 0x9a, hi: 0x9b}, + {value: 0x812d, lo: 0x9c, hi: 0x9f}, + {value: 0x8132, lo: 0xa0, hi: 0xa0}, + {value: 0x8101, lo: 0xa2, hi: 0xa8}, + {value: 0x812d, lo: 0xad, hi: 0xad}, + {value: 0x8132, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb8, hi: 0xb9}, + // Block 0x3c, offset 0x147 + {value: 0x0000, lo: 0x0f}, + {value: 0x8132, lo: 0x80, hi: 0x81}, + {value: 0x812d, lo: 0x82, hi: 0x82}, + {value: 0x8132, lo: 0x83, hi: 0x89}, + {value: 0x812d, lo: 0x8a, hi: 0x8a}, + {value: 0x8132, lo: 0x8b, hi: 0x8c}, + {value: 0x8135, lo: 0x8d, hi: 0x8d}, + {value: 0x812a, lo: 0x8e, hi: 0x8e}, + {value: 0x812d, lo: 0x8f, hi: 0x8f}, + {value: 0x8129, lo: 0x90, hi: 0x90}, + {value: 0x8132, lo: 0x91, hi: 0xb5}, + {value: 0x8132, lo: 0xbb, hi: 0xbb}, + {value: 0x8134, lo: 0xbc, hi: 0xbc}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + {value: 0x8132, lo: 0xbe, hi: 0xbe}, + {value: 0x812d, lo: 0xbf, hi: 0xbf}, + // Block 0x3d, offset 0x157 + {value: 0x0004, lo: 0x03}, + {value: 0x0433, lo: 0x80, hi: 0x81}, + {value: 0x8100, lo: 0x97, hi: 0x97}, + {value: 0x8100, lo: 0xbe, hi: 0xbe}, + // Block 0x3e, offset 0x15b + {value: 0x0000, lo: 0x0d}, + {value: 0x8132, lo: 0x90, hi: 0x91}, + {value: 0x8101, lo: 0x92, hi: 0x93}, + {value: 0x8132, lo: 0x94, hi: 0x97}, + {value: 0x8101, lo: 0x98, hi: 0x9a}, + {value: 0x8132, lo: 0x9b, hi: 0x9c}, + {value: 0x8132, lo: 0xa1, hi: 0xa1}, + {value: 0x8101, lo: 0xa5, hi: 0xa6}, + {value: 0x8132, lo: 0xa7, hi: 0xa7}, + {value: 0x812d, lo: 0xa8, hi: 0xa8}, + {value: 0x8132, lo: 0xa9, hi: 0xa9}, + {value: 0x8101, lo: 0xaa, hi: 0xab}, + {value: 0x812d, lo: 0xac, hi: 0xaf}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + // Block 0x3f, offset 0x169 + {value: 0x427b, lo: 0x02}, + {value: 0x01b8, lo: 0xa6, hi: 0xa6}, + {value: 0x0057, lo: 0xaa, hi: 0xab}, + // Block 0x40, offset 0x16c + {value: 0x0007, lo: 0x05}, + {value: 0xa000, lo: 0x90, hi: 0x90}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0xa000, lo: 0x94, hi: 0x94}, + {value: 0x3bb9, lo: 0x9a, hi: 0x9b}, + {value: 0x3bc7, lo: 0xae, hi: 0xae}, + // Block 0x41, offset 0x172 + {value: 0x000e, lo: 0x05}, + {value: 0x3bce, lo: 0x8d, hi: 0x8e}, + {value: 0x3bd5, lo: 0x8f, hi: 0x8f}, + {value: 0xa000, lo: 0x90, hi: 0x90}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0xa000, lo: 0x94, hi: 0x94}, + // Block 0x42, offset 0x178 + {value: 0x6408, lo: 0x0a}, + {value: 0xa000, lo: 0x83, hi: 0x83}, + {value: 0x3be3, lo: 0x84, hi: 0x84}, + {value: 0xa000, lo: 0x88, hi: 0x88}, + {value: 0x3bea, lo: 0x89, hi: 0x89}, + {value: 0xa000, lo: 0x8b, hi: 0x8b}, + {value: 0x3bf1, lo: 0x8c, hi: 0x8c}, + {value: 0xa000, lo: 0xa3, hi: 0xa3}, + {value: 0x3bf8, lo: 0xa4, hi: 0xa5}, + {value: 0x3bff, lo: 0xa6, hi: 0xa6}, + {value: 0xa000, lo: 0xbc, hi: 0xbc}, + // Block 0x43, offset 0x183 + {value: 0x0007, lo: 0x03}, + {value: 0x3c68, lo: 0xa0, hi: 0xa1}, + {value: 0x3c92, lo: 0xa2, hi: 0xa3}, + {value: 0x3cbc, lo: 0xaa, hi: 0xad}, + // Block 0x44, offset 0x187 + {value: 0x0004, lo: 0x01}, + {value: 0x048b, lo: 0xa9, hi: 0xaa}, + // Block 0x45, offset 0x189 + {value: 0x0000, lo: 0x01}, + {value: 0x4573, lo: 0x9c, hi: 0x9c}, + // Block 0x46, offset 0x18b + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xaf, hi: 0xb1}, + // Block 0x47, offset 0x18d + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x48, offset 0x18f + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xa0, hi: 0xbf}, + // Block 0x49, offset 0x191 + {value: 0x0000, lo: 0x05}, + {value: 0x812c, lo: 0xaa, hi: 0xaa}, + {value: 0x8131, lo: 0xab, hi: 0xab}, + {value: 0x8133, lo: 0xac, hi: 0xac}, + {value: 0x812e, lo: 0xad, hi: 0xad}, + {value: 0x812f, lo: 0xae, hi: 0xaf}, + // Block 0x4a, offset 0x197 + {value: 0x0000, lo: 0x03}, + {value: 0x4a81, lo: 0xb3, hi: 0xb3}, + {value: 0x4a81, lo: 0xb5, hi: 0xb6}, + {value: 0x4a81, lo: 0xba, hi: 0xbf}, + // Block 0x4b, offset 0x19b + {value: 0x0000, lo: 0x01}, + {value: 0x4a81, lo: 0x8f, hi: 0xa3}, + // Block 0x4c, offset 0x19d + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0xae, hi: 0xbe}, + // Block 0x4d, offset 0x19f + {value: 0x0000, lo: 0x07}, + {value: 0x8100, lo: 0x84, hi: 0x84}, + {value: 0x8100, lo: 0x87, hi: 0x87}, + {value: 0x8100, lo: 0x90, hi: 0x90}, + {value: 0x8100, lo: 0x9e, hi: 0x9e}, + {value: 0x8100, lo: 0xa1, hi: 0xa1}, + {value: 0x8100, lo: 0xb2, hi: 0xb2}, + {value: 0x8100, lo: 0xbb, hi: 0xbb}, + // Block 0x4e, offset 0x1a7 + {value: 0x0000, lo: 0x03}, + {value: 0x8100, lo: 0x80, hi: 0x80}, + {value: 0x8100, lo: 0x8b, hi: 0x8b}, + {value: 0x8100, lo: 0x8e, hi: 0x8e}, + // Block 0x4f, offset 0x1ab + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0xaf, hi: 0xaf}, + {value: 0x8132, lo: 0xb4, hi: 0xbd}, + // Block 0x50, offset 0x1ae + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x9e, hi: 0x9f}, + // Block 0x51, offset 0x1b0 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb0, hi: 0xb1}, + // Block 0x52, offset 0x1b2 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x86, hi: 0x86}, + // Block 0x53, offset 0x1b4 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0xa0, hi: 0xb1}, + // Block 0x54, offset 0x1b7 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xab, hi: 0xad}, + // Block 0x55, offset 0x1b9 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x93, hi: 0x93}, + // Block 0x56, offset 0x1bb + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xb3, hi: 0xb3}, + // Block 0x57, offset 0x1bd + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x80, hi: 0x80}, + // Block 0x58, offset 0x1bf + {value: 0x0000, lo: 0x05}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + {value: 0x8132, lo: 0xb2, hi: 0xb3}, + {value: 0x812d, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb7, hi: 0xb8}, + {value: 0x8132, lo: 0xbe, hi: 0xbf}, + // Block 0x59, offset 0x1c5 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x81, hi: 0x81}, + {value: 0x8104, lo: 0xb6, hi: 0xb6}, + // Block 0x5a, offset 0x1c8 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xad, hi: 0xad}, + // Block 0x5b, offset 0x1ca + {value: 0x0000, lo: 0x06}, + {value: 0xe500, lo: 0x80, hi: 0x80}, + {value: 0xc600, lo: 0x81, hi: 0x9b}, + {value: 0xe500, lo: 0x9c, hi: 0x9c}, + {value: 0xc600, lo: 0x9d, hi: 0xb7}, + {value: 0xe500, lo: 0xb8, hi: 0xb8}, + {value: 0xc600, lo: 0xb9, hi: 0xbf}, + // Block 0x5c, offset 0x1d1 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x93}, + {value: 0xe500, lo: 0x94, hi: 0x94}, + {value: 0xc600, lo: 0x95, hi: 0xaf}, + {value: 0xe500, lo: 0xb0, hi: 0xb0}, + {value: 0xc600, lo: 0xb1, hi: 0xbf}, + // Block 0x5d, offset 0x1d7 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x8b}, + {value: 0xe500, lo: 0x8c, hi: 0x8c}, + {value: 0xc600, lo: 0x8d, hi: 0xa7}, + {value: 0xe500, lo: 0xa8, hi: 0xa8}, + {value: 0xc600, lo: 0xa9, hi: 0xbf}, + // Block 0x5e, offset 0x1dd + {value: 0x0000, lo: 0x07}, + {value: 0xc600, lo: 0x80, hi: 0x83}, + {value: 0xe500, lo: 0x84, hi: 0x84}, + {value: 0xc600, lo: 0x85, hi: 0x9f}, + {value: 0xe500, lo: 0xa0, hi: 0xa0}, + {value: 0xc600, lo: 0xa1, hi: 0xbb}, + {value: 0xe500, lo: 0xbc, hi: 0xbc}, + {value: 0xc600, lo: 0xbd, hi: 0xbf}, + // Block 0x5f, offset 0x1e5 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x97}, + {value: 0xe500, lo: 0x98, hi: 0x98}, + {value: 0xc600, lo: 0x99, hi: 0xb3}, + {value: 0xe500, lo: 0xb4, hi: 0xb4}, + {value: 0xc600, lo: 0xb5, hi: 0xbf}, + // Block 0x60, offset 0x1eb + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x8f}, + {value: 0xe500, lo: 0x90, hi: 0x90}, + {value: 0xc600, lo: 0x91, hi: 0xab}, + {value: 0xe500, lo: 0xac, hi: 0xac}, + {value: 0xc600, lo: 0xad, hi: 0xbf}, + // Block 0x61, offset 0x1f1 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x87}, + {value: 0xe500, lo: 0x88, hi: 0x88}, + {value: 0xc600, lo: 0x89, hi: 0xa3}, + {value: 0xe500, lo: 0xa4, hi: 0xa4}, + {value: 0xc600, lo: 0xa5, hi: 0xbf}, + // Block 0x62, offset 0x1f7 + {value: 0x0000, lo: 0x03}, + {value: 0xc600, lo: 0x80, hi: 0x87}, + {value: 0xe500, lo: 0x88, hi: 0x88}, + {value: 0xc600, lo: 0x89, hi: 0xa3}, + // Block 0x63, offset 0x1fb + {value: 0x0006, lo: 0x0d}, + {value: 0x4426, lo: 0x9d, hi: 0x9d}, + {value: 0x8115, lo: 0x9e, hi: 0x9e}, + {value: 0x4498, lo: 0x9f, hi: 0x9f}, + {value: 0x4486, lo: 0xaa, hi: 0xab}, + {value: 0x458a, lo: 0xac, hi: 0xac}, + {value: 0x4592, lo: 0xad, hi: 0xad}, + {value: 0x43de, lo: 0xae, hi: 0xb1}, + {value: 0x43fc, lo: 0xb2, hi: 0xb4}, + {value: 0x4414, lo: 0xb5, hi: 0xb6}, + {value: 0x4420, lo: 0xb8, hi: 0xb8}, + {value: 0x442c, lo: 0xb9, hi: 0xbb}, + {value: 0x4444, lo: 0xbc, hi: 0xbc}, + {value: 0x444a, lo: 0xbe, hi: 0xbe}, + // Block 0x64, offset 0x209 + {value: 0x0006, lo: 0x08}, + {value: 0x4450, lo: 0x80, hi: 0x81}, + {value: 0x445c, lo: 0x83, hi: 0x84}, + {value: 0x446e, lo: 0x86, hi: 0x89}, + {value: 0x4492, lo: 0x8a, hi: 0x8a}, + {value: 0x440e, lo: 0x8b, hi: 0x8b}, + {value: 0x43f6, lo: 0x8c, hi: 0x8c}, + {value: 0x443e, lo: 0x8d, hi: 0x8d}, + {value: 0x4468, lo: 0x8e, hi: 0x8e}, + // Block 0x65, offset 0x212 + {value: 0x0000, lo: 0x02}, + {value: 0x8100, lo: 0xa4, hi: 0xa5}, + {value: 0x8100, lo: 0xb0, hi: 0xb1}, + // Block 0x66, offset 0x215 + {value: 0x0000, lo: 0x02}, + {value: 0x8100, lo: 0x9b, hi: 0x9d}, + {value: 0x8200, lo: 0x9e, hi: 0xa3}, + // Block 0x67, offset 0x218 + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0x90, hi: 0x90}, + // Block 0x68, offset 0x21a + {value: 0x0000, lo: 0x02}, + {value: 0x8100, lo: 0x99, hi: 0x99}, + {value: 0x8200, lo: 0xb2, hi: 0xb4}, + // Block 0x69, offset 0x21d + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0xbc, hi: 0xbd}, + // Block 0x6a, offset 0x21f + {value: 0x0000, lo: 0x03}, + {value: 0x8132, lo: 0xa0, hi: 0xa6}, + {value: 0x812d, lo: 0xa7, hi: 0xad}, + {value: 0x8132, lo: 0xae, hi: 0xaf}, + // Block 0x6b, offset 0x223 + {value: 0x0000, lo: 0x04}, + {value: 0x8100, lo: 0x89, hi: 0x8c}, + {value: 0x8100, lo: 0xb0, hi: 0xb2}, + {value: 0x8100, lo: 0xb4, hi: 0xb4}, + {value: 0x8100, lo: 0xb6, hi: 0xbf}, + // Block 0x6c, offset 0x228 + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0x81, hi: 0x8c}, + // Block 0x6d, offset 0x22a + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0xb5, hi: 0xba}, + // Block 0x6e, offset 0x22c + {value: 0x0000, lo: 0x04}, + {value: 0x4a81, lo: 0x9e, hi: 0x9f}, + {value: 0x4a81, lo: 0xa3, hi: 0xa3}, + {value: 0x4a81, lo: 0xa5, hi: 0xa6}, + {value: 0x4a81, lo: 0xaa, hi: 0xaf}, + // Block 0x6f, offset 0x231 + {value: 0x0000, lo: 0x05}, + {value: 0x4a81, lo: 0x82, hi: 0x87}, + {value: 0x4a81, lo: 0x8a, hi: 0x8f}, + {value: 0x4a81, lo: 0x92, hi: 0x97}, + {value: 0x4a81, lo: 0x9a, hi: 0x9c}, + {value: 0x8100, lo: 0xa3, hi: 0xa3}, + // Block 0x70, offset 0x237 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + // Block 0x71, offset 0x239 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xa0, hi: 0xa0}, + // Block 0x72, offset 0x23b + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb6, hi: 0xba}, + // Block 0x73, offset 0x23d + {value: 0x002c, lo: 0x05}, + {value: 0x812d, lo: 0x8d, hi: 0x8d}, + {value: 0x8132, lo: 0x8f, hi: 0x8f}, + {value: 0x8132, lo: 0xb8, hi: 0xb8}, + {value: 0x8101, lo: 0xb9, hi: 0xba}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x74, offset 0x243 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0xa5, hi: 0xa5}, + {value: 0x812d, lo: 0xa6, hi: 0xa6}, + // Block 0x75, offset 0x246 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x86, hi: 0x86}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x76, offset 0x249 + {value: 0x17fe, lo: 0x07}, + {value: 0xa000, lo: 0x99, hi: 0x99}, + {value: 0x4238, lo: 0x9a, hi: 0x9a}, + {value: 0xa000, lo: 0x9b, hi: 0x9b}, + {value: 0x4242, lo: 0x9c, hi: 0x9c}, + {value: 0xa000, lo: 0xa5, hi: 0xa5}, + {value: 0x424c, lo: 0xab, hi: 0xab}, + {value: 0x8104, lo: 0xb9, hi: 0xba}, + // Block 0x77, offset 0x251 + {value: 0x0000, lo: 0x06}, + {value: 0x8132, lo: 0x80, hi: 0x82}, + {value: 0x9900, lo: 0xa7, hi: 0xa7}, + {value: 0x2d7e, lo: 0xae, hi: 0xae}, + {value: 0x2d88, lo: 0xaf, hi: 0xaf}, + {value: 0xa000, lo: 0xb1, hi: 0xb2}, + {value: 0x8104, lo: 0xb3, hi: 0xb4}, + // Block 0x78, offset 0x258 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x80, hi: 0x80}, + {value: 0x8102, lo: 0x8a, hi: 0x8a}, + // Block 0x79, offset 0x25b + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xb5, hi: 0xb5}, + {value: 0x8102, lo: 0xb6, hi: 0xb6}, + // Block 0x7a, offset 0x25e + {value: 0x0002, lo: 0x01}, + {value: 0x8102, lo: 0xa9, hi: 0xaa}, + // Block 0x7b, offset 0x260 + {value: 0x0000, lo: 0x07}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2d92, lo: 0x8b, hi: 0x8b}, + {value: 0x2d9c, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + {value: 0x8132, lo: 0xa6, hi: 0xac}, + {value: 0x8132, lo: 0xb0, hi: 0xb4}, + // Block 0x7c, offset 0x268 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x82, hi: 0x82}, + {value: 0x8102, lo: 0x86, hi: 0x86}, + // Block 0x7d, offset 0x26b + {value: 0x6b5a, lo: 0x06}, + {value: 0x9900, lo: 0xb0, hi: 0xb0}, + {value: 0xa000, lo: 0xb9, hi: 0xb9}, + {value: 0x9900, lo: 0xba, hi: 0xba}, + {value: 0x2db0, lo: 0xbb, hi: 0xbb}, + {value: 0x2da6, lo: 0xbc, hi: 0xbd}, + {value: 0x2dba, lo: 0xbe, hi: 0xbe}, + // Block 0x7e, offset 0x272 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x82, hi: 0x82}, + {value: 0x8102, lo: 0x83, hi: 0x83}, + // Block 0x7f, offset 0x275 + {value: 0x0000, lo: 0x05}, + {value: 0x9900, lo: 0xaf, hi: 0xaf}, + {value: 0xa000, lo: 0xb8, hi: 0xb9}, + {value: 0x2dc4, lo: 0xba, hi: 0xba}, + {value: 0x2dce, lo: 0xbb, hi: 0xbb}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x80, offset 0x27b + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0x80, hi: 0x80}, + // Block 0x81, offset 0x27d + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xb6, hi: 0xb6}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + // Block 0x82, offset 0x280 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xab, hi: 0xab}, + // Block 0x83, offset 0x282 + {value: 0x0000, lo: 0x01}, + {value: 0x8101, lo: 0xb0, hi: 0xb4}, + // Block 0x84, offset 0x284 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb0, hi: 0xb6}, + // Block 0x85, offset 0x286 + {value: 0x0000, lo: 0x01}, + {value: 0x8101, lo: 0x9e, hi: 0x9e}, + // Block 0x86, offset 0x288 + {value: 0x0000, lo: 0x0c}, + {value: 0x4662, lo: 0x9e, hi: 0x9e}, + {value: 0x466c, lo: 0x9f, hi: 0x9f}, + {value: 0x46a0, lo: 0xa0, hi: 0xa0}, + {value: 0x46ae, lo: 0xa1, hi: 0xa1}, + {value: 0x46bc, lo: 0xa2, hi: 0xa2}, + {value: 0x46ca, lo: 0xa3, hi: 0xa3}, + {value: 0x46d8, lo: 0xa4, hi: 0xa4}, + {value: 0x812b, lo: 0xa5, hi: 0xa6}, + {value: 0x8101, lo: 0xa7, hi: 0xa9}, + {value: 0x8130, lo: 0xad, hi: 0xad}, + {value: 0x812b, lo: 0xae, hi: 0xb2}, + {value: 0x812d, lo: 0xbb, hi: 0xbf}, + // Block 0x87, offset 0x295 + {value: 0x0000, lo: 0x09}, + {value: 0x812d, lo: 0x80, hi: 0x82}, + {value: 0x8132, lo: 0x85, hi: 0x89}, + {value: 0x812d, lo: 0x8a, hi: 0x8b}, + {value: 0x8132, lo: 0xaa, hi: 0xad}, + {value: 0x4676, lo: 0xbb, hi: 0xbb}, + {value: 0x4680, lo: 0xbc, hi: 0xbc}, + {value: 0x46e6, lo: 0xbd, hi: 0xbd}, + {value: 0x4702, lo: 0xbe, hi: 0xbe}, + {value: 0x46f4, lo: 0xbf, hi: 0xbf}, + // Block 0x88, offset 0x29f + {value: 0x0000, lo: 0x01}, + {value: 0x4710, lo: 0x80, hi: 0x80}, + // Block 0x89, offset 0x2a1 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x82, hi: 0x84}, + // Block 0x8a, offset 0x2a3 + {value: 0x0000, lo: 0x05}, + {value: 0x8132, lo: 0x80, hi: 0x86}, + {value: 0x8132, lo: 0x88, hi: 0x98}, + {value: 0x8132, lo: 0x9b, hi: 0xa1}, + {value: 0x8132, lo: 0xa3, hi: 0xa4}, + {value: 0x8132, lo: 0xa6, hi: 0xaa}, + // Block 0x8b, offset 0x2a9 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x90, hi: 0x96}, + // Block 0x8c, offset 0x2ab + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x84, hi: 0x89}, + {value: 0x8102, lo: 0x8a, hi: 0x8a}, + // Block 0x8d, offset 0x2ae + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0x93, hi: 0x93}, +} + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *nfkcTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return nfkcValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfkcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfkcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = nfkcIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *nfkcTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return nfkcValues[c0] + } + i := nfkcIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = nfkcIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = nfkcIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *nfkcTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return nfkcValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfkcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfkcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = nfkcIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *nfkcTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return nfkcValues[c0] + } + i := nfkcIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = nfkcIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = nfkcIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// nfkcTrie. Total size: 16994 bytes (16.60 KiB). Checksum: 146925fc21092b17. +type nfkcTrie struct{} + +func newNfkcTrie(i int) *nfkcTrie { + return &nfkcTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *nfkcTrie) lookupValue(n uint32, b byte) uint16 { + switch { + case n < 90: + return uint16(nfkcValues[n<<6+uint32(b)]) + default: + n -= 90 + return uint16(nfkcSparse.lookup(n, b)) + } +} + +// nfkcValues: 92 blocks, 5888 entries, 11776 bytes +// The third block is the zero block. +var nfkcValues = [5888]uint16{ + // Block 0x0, offset 0x0 + 0x3c: 0xa000, 0x3d: 0xa000, 0x3e: 0xa000, + // Block 0x1, offset 0x40 + 0x41: 0xa000, 0x42: 0xa000, 0x43: 0xa000, 0x44: 0xa000, 0x45: 0xa000, + 0x46: 0xa000, 0x47: 0xa000, 0x48: 0xa000, 0x49: 0xa000, 0x4a: 0xa000, 0x4b: 0xa000, + 0x4c: 0xa000, 0x4d: 0xa000, 0x4e: 0xa000, 0x4f: 0xa000, 0x50: 0xa000, + 0x52: 0xa000, 0x53: 0xa000, 0x54: 0xa000, 0x55: 0xa000, 0x56: 0xa000, 0x57: 0xa000, + 0x58: 0xa000, 0x59: 0xa000, 0x5a: 0xa000, + 0x61: 0xa000, 0x62: 0xa000, 0x63: 0xa000, + 0x64: 0xa000, 0x65: 0xa000, 0x66: 0xa000, 0x67: 0xa000, 0x68: 0xa000, 0x69: 0xa000, + 0x6a: 0xa000, 0x6b: 0xa000, 0x6c: 0xa000, 0x6d: 0xa000, 0x6e: 0xa000, 0x6f: 0xa000, + 0x70: 0xa000, 0x72: 0xa000, 0x73: 0xa000, 0x74: 0xa000, 0x75: 0xa000, + 0x76: 0xa000, 0x77: 0xa000, 0x78: 0xa000, 0x79: 0xa000, 0x7a: 0xa000, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc0: 0x2f6f, 0xc1: 0x2f74, 0xc2: 0x471e, 0xc3: 0x2f79, 0xc4: 0x472d, 0xc5: 0x4732, + 0xc6: 0xa000, 0xc7: 0x473c, 0xc8: 0x2fe2, 0xc9: 0x2fe7, 0xca: 0x4741, 0xcb: 0x2ffb, + 0xcc: 0x306e, 0xcd: 0x3073, 0xce: 0x3078, 0xcf: 0x4755, 0xd1: 0x3104, + 0xd2: 0x3127, 0xd3: 0x312c, 0xd4: 0x475f, 0xd5: 0x4764, 0xd6: 0x4773, + 0xd8: 0xa000, 0xd9: 0x31b3, 0xda: 0x31b8, 0xdb: 0x31bd, 0xdc: 0x47a5, 0xdd: 0x3235, + 0xe0: 0x327b, 0xe1: 0x3280, 0xe2: 0x47af, 0xe3: 0x3285, + 0xe4: 0x47be, 0xe5: 0x47c3, 0xe6: 0xa000, 0xe7: 0x47cd, 0xe8: 0x32ee, 0xe9: 0x32f3, + 0xea: 0x47d2, 0xeb: 0x3307, 0xec: 0x337f, 0xed: 0x3384, 0xee: 0x3389, 0xef: 0x47e6, + 0xf1: 0x3415, 0xf2: 0x3438, 0xf3: 0x343d, 0xf4: 0x47f0, 0xf5: 0x47f5, + 0xf6: 0x4804, 0xf8: 0xa000, 0xf9: 0x34c9, 0xfa: 0x34ce, 0xfb: 0x34d3, + 0xfc: 0x4836, 0xfd: 0x3550, 0xff: 0x3569, + // Block 0x4, offset 0x100 + 0x100: 0x2f7e, 0x101: 0x328a, 0x102: 0x4723, 0x103: 0x47b4, 0x104: 0x2f9c, 0x105: 0x32a8, + 0x106: 0x2fb0, 0x107: 0x32bc, 0x108: 0x2fb5, 0x109: 0x32c1, 0x10a: 0x2fba, 0x10b: 0x32c6, + 0x10c: 0x2fbf, 0x10d: 0x32cb, 0x10e: 0x2fc9, 0x10f: 0x32d5, + 0x112: 0x4746, 0x113: 0x47d7, 0x114: 0x2ff1, 0x115: 0x32fd, 0x116: 0x2ff6, 0x117: 0x3302, + 0x118: 0x3014, 0x119: 0x3320, 0x11a: 0x3005, 0x11b: 0x3311, 0x11c: 0x302d, 0x11d: 0x3339, + 0x11e: 0x3037, 0x11f: 0x3343, 0x120: 0x303c, 0x121: 0x3348, 0x122: 0x3046, 0x123: 0x3352, + 0x124: 0x304b, 0x125: 0x3357, 0x128: 0x307d, 0x129: 0x338e, + 0x12a: 0x3082, 0x12b: 0x3393, 0x12c: 0x3087, 0x12d: 0x3398, 0x12e: 0x30aa, 0x12f: 0x33b6, + 0x130: 0x308c, 0x132: 0x195d, 0x133: 0x19e7, 0x134: 0x30b4, 0x135: 0x33c0, + 0x136: 0x30c8, 0x137: 0x33d9, 0x139: 0x30d2, 0x13a: 0x33e3, 0x13b: 0x30dc, + 0x13c: 0x33ed, 0x13d: 0x30d7, 0x13e: 0x33e8, 0x13f: 0x1bac, + // Block 0x5, offset 0x140 + 0x140: 0x1c34, 0x143: 0x30ff, 0x144: 0x3410, 0x145: 0x3118, + 0x146: 0x3429, 0x147: 0x310e, 0x148: 0x341f, 0x149: 0x1c5c, + 0x14c: 0x4769, 0x14d: 0x47fa, 0x14e: 0x3131, 0x14f: 0x3442, 0x150: 0x313b, 0x151: 0x344c, + 0x154: 0x3159, 0x155: 0x346a, 0x156: 0x3172, 0x157: 0x3483, + 0x158: 0x3163, 0x159: 0x3474, 0x15a: 0x478c, 0x15b: 0x481d, 0x15c: 0x317c, 0x15d: 0x348d, + 0x15e: 0x318b, 0x15f: 0x349c, 0x160: 0x4791, 0x161: 0x4822, 0x162: 0x31a4, 0x163: 0x34ba, + 0x164: 0x3195, 0x165: 0x34ab, 0x168: 0x479b, 0x169: 0x482c, + 0x16a: 0x47a0, 0x16b: 0x4831, 0x16c: 0x31c2, 0x16d: 0x34d8, 0x16e: 0x31cc, 0x16f: 0x34e2, + 0x170: 0x31d1, 0x171: 0x34e7, 0x172: 0x31ef, 0x173: 0x3505, 0x174: 0x3212, 0x175: 0x3528, + 0x176: 0x323a, 0x177: 0x3555, 0x178: 0x324e, 0x179: 0x325d, 0x17a: 0x357d, 0x17b: 0x3267, + 0x17c: 0x3587, 0x17d: 0x326c, 0x17e: 0x358c, 0x17f: 0x00a7, + // Block 0x6, offset 0x180 + 0x184: 0x2dee, 0x185: 0x2df4, + 0x186: 0x2dfa, 0x187: 0x1972, 0x188: 0x1975, 0x189: 0x1a08, 0x18a: 0x1987, 0x18b: 0x198a, + 0x18c: 0x1a3e, 0x18d: 0x2f88, 0x18e: 0x3294, 0x18f: 0x3096, 0x190: 0x33a2, 0x191: 0x3140, + 0x192: 0x3451, 0x193: 0x31d6, 0x194: 0x34ec, 0x195: 0x39cf, 0x196: 0x3b5e, 0x197: 0x39c8, + 0x198: 0x3b57, 0x199: 0x39d6, 0x19a: 0x3b65, 0x19b: 0x39c1, 0x19c: 0x3b50, + 0x19e: 0x38b0, 0x19f: 0x3a3f, 0x1a0: 0x38a9, 0x1a1: 0x3a38, 0x1a2: 0x35b3, 0x1a3: 0x35c5, + 0x1a6: 0x3041, 0x1a7: 0x334d, 0x1a8: 0x30be, 0x1a9: 0x33cf, + 0x1aa: 0x4782, 0x1ab: 0x4813, 0x1ac: 0x3990, 0x1ad: 0x3b1f, 0x1ae: 0x35d7, 0x1af: 0x35dd, + 0x1b0: 0x33c5, 0x1b1: 0x1942, 0x1b2: 0x1945, 0x1b3: 0x19cf, 0x1b4: 0x3028, 0x1b5: 0x3334, + 0x1b8: 0x30fa, 0x1b9: 0x340b, 0x1ba: 0x38b7, 0x1bb: 0x3a46, + 0x1bc: 0x35ad, 0x1bd: 0x35bf, 0x1be: 0x35b9, 0x1bf: 0x35cb, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x2f8d, 0x1c1: 0x3299, 0x1c2: 0x2f92, 0x1c3: 0x329e, 0x1c4: 0x300a, 0x1c5: 0x3316, + 0x1c6: 0x300f, 0x1c7: 0x331b, 0x1c8: 0x309b, 0x1c9: 0x33a7, 0x1ca: 0x30a0, 0x1cb: 0x33ac, + 0x1cc: 0x3145, 0x1cd: 0x3456, 0x1ce: 0x314a, 0x1cf: 0x345b, 0x1d0: 0x3168, 0x1d1: 0x3479, + 0x1d2: 0x316d, 0x1d3: 0x347e, 0x1d4: 0x31db, 0x1d5: 0x34f1, 0x1d6: 0x31e0, 0x1d7: 0x34f6, + 0x1d8: 0x3186, 0x1d9: 0x3497, 0x1da: 0x319f, 0x1db: 0x34b5, + 0x1de: 0x305a, 0x1df: 0x3366, + 0x1e6: 0x4728, 0x1e7: 0x47b9, 0x1e8: 0x4750, 0x1e9: 0x47e1, + 0x1ea: 0x395f, 0x1eb: 0x3aee, 0x1ec: 0x393c, 0x1ed: 0x3acb, 0x1ee: 0x476e, 0x1ef: 0x47ff, + 0x1f0: 0x3958, 0x1f1: 0x3ae7, 0x1f2: 0x3244, 0x1f3: 0x355f, + // Block 0x8, offset 0x200 + 0x200: 0x9932, 0x201: 0x9932, 0x202: 0x9932, 0x203: 0x9932, 0x204: 0x9932, 0x205: 0x8132, + 0x206: 0x9932, 0x207: 0x9932, 0x208: 0x9932, 0x209: 0x9932, 0x20a: 0x9932, 0x20b: 0x9932, + 0x20c: 0x9932, 0x20d: 0x8132, 0x20e: 0x8132, 0x20f: 0x9932, 0x210: 0x8132, 0x211: 0x9932, + 0x212: 0x8132, 0x213: 0x9932, 0x214: 0x9932, 0x215: 0x8133, 0x216: 0x812d, 0x217: 0x812d, + 0x218: 0x812d, 0x219: 0x812d, 0x21a: 0x8133, 0x21b: 0x992b, 0x21c: 0x812d, 0x21d: 0x812d, + 0x21e: 0x812d, 0x21f: 0x812d, 0x220: 0x812d, 0x221: 0x8129, 0x222: 0x8129, 0x223: 0x992d, + 0x224: 0x992d, 0x225: 0x992d, 0x226: 0x992d, 0x227: 0x9929, 0x228: 0x9929, 0x229: 0x812d, + 0x22a: 0x812d, 0x22b: 0x812d, 0x22c: 0x812d, 0x22d: 0x992d, 0x22e: 0x992d, 0x22f: 0x812d, + 0x230: 0x992d, 0x231: 0x992d, 0x232: 0x812d, 0x233: 0x812d, 0x234: 0x8101, 0x235: 0x8101, + 0x236: 0x8101, 0x237: 0x8101, 0x238: 0x9901, 0x239: 0x812d, 0x23a: 0x812d, 0x23b: 0x812d, + 0x23c: 0x812d, 0x23d: 0x8132, 0x23e: 0x8132, 0x23f: 0x8132, + // Block 0x9, offset 0x240 + 0x240: 0x4a44, 0x241: 0x4a49, 0x242: 0x9932, 0x243: 0x4a4e, 0x244: 0x4a53, 0x245: 0x9936, + 0x246: 0x8132, 0x247: 0x812d, 0x248: 0x812d, 0x249: 0x812d, 0x24a: 0x8132, 0x24b: 0x8132, + 0x24c: 0x8132, 0x24d: 0x812d, 0x24e: 0x812d, 0x250: 0x8132, 0x251: 0x8132, + 0x252: 0x8132, 0x253: 0x812d, 0x254: 0x812d, 0x255: 0x812d, 0x256: 0x812d, 0x257: 0x8132, + 0x258: 0x8133, 0x259: 0x812d, 0x25a: 0x812d, 0x25b: 0x8132, 0x25c: 0x8134, 0x25d: 0x8135, + 0x25e: 0x8135, 0x25f: 0x8134, 0x260: 0x8135, 0x261: 0x8135, 0x262: 0x8134, 0x263: 0x8132, + 0x264: 0x8132, 0x265: 0x8132, 0x266: 0x8132, 0x267: 0x8132, 0x268: 0x8132, 0x269: 0x8132, + 0x26a: 0x8132, 0x26b: 0x8132, 0x26c: 0x8132, 0x26d: 0x8132, 0x26e: 0x8132, 0x26f: 0x8132, + 0x274: 0x0170, + 0x27a: 0x42a5, + 0x27e: 0x0037, + // Block 0xa, offset 0x280 + 0x284: 0x425a, 0x285: 0x4511, + 0x286: 0x35e9, 0x287: 0x00ce, 0x288: 0x3607, 0x289: 0x3613, 0x28a: 0x3625, + 0x28c: 0x3643, 0x28e: 0x3655, 0x28f: 0x3673, 0x290: 0x3e08, 0x291: 0xa000, + 0x295: 0xa000, 0x297: 0xa000, + 0x299: 0xa000, + 0x29f: 0xa000, 0x2a1: 0xa000, + 0x2a5: 0xa000, 0x2a9: 0xa000, + 0x2aa: 0x3637, 0x2ab: 0x3667, 0x2ac: 0x4894, 0x2ad: 0x3697, 0x2ae: 0x48be, 0x2af: 0x36a9, + 0x2b0: 0x3e70, 0x2b1: 0xa000, 0x2b5: 0xa000, + 0x2b7: 0xa000, 0x2b9: 0xa000, + 0x2bf: 0xa000, + // Block 0xb, offset 0x2c0 + 0x2c1: 0xa000, 0x2c5: 0xa000, + 0x2c9: 0xa000, 0x2ca: 0x48d6, 0x2cb: 0x48f4, + 0x2cc: 0x36c7, 0x2cd: 0x36df, 0x2ce: 0x490c, 0x2d0: 0x01be, 0x2d1: 0x01d0, + 0x2d2: 0x01ac, 0x2d3: 0x43a2, 0x2d4: 0x43a8, 0x2d5: 0x01fa, 0x2d6: 0x01e8, + 0x2f0: 0x01d6, 0x2f1: 0x01eb, 0x2f2: 0x01ee, 0x2f4: 0x0188, 0x2f5: 0x01c7, + 0x2f9: 0x01a6, + // Block 0xc, offset 0x300 + 0x300: 0x3721, 0x301: 0x372d, 0x303: 0x371b, + 0x306: 0xa000, 0x307: 0x3709, + 0x30c: 0x375d, 0x30d: 0x3745, 0x30e: 0x376f, 0x310: 0xa000, + 0x313: 0xa000, 0x315: 0xa000, 0x316: 0xa000, 0x317: 0xa000, + 0x318: 0xa000, 0x319: 0x3751, 0x31a: 0xa000, + 0x31e: 0xa000, 0x323: 0xa000, + 0x327: 0xa000, + 0x32b: 0xa000, 0x32d: 0xa000, + 0x330: 0xa000, 0x333: 0xa000, 0x335: 0xa000, + 0x336: 0xa000, 0x337: 0xa000, 0x338: 0xa000, 0x339: 0x37d5, 0x33a: 0xa000, + 0x33e: 0xa000, + // Block 0xd, offset 0x340 + 0x341: 0x3733, 0x342: 0x37b7, + 0x350: 0x370f, 0x351: 0x3793, + 0x352: 0x3715, 0x353: 0x3799, 0x356: 0x3727, 0x357: 0x37ab, + 0x358: 0xa000, 0x359: 0xa000, 0x35a: 0x3829, 0x35b: 0x382f, 0x35c: 0x3739, 0x35d: 0x37bd, + 0x35e: 0x373f, 0x35f: 0x37c3, 0x362: 0x374b, 0x363: 0x37cf, + 0x364: 0x3757, 0x365: 0x37db, 0x366: 0x3763, 0x367: 0x37e7, 0x368: 0xa000, 0x369: 0xa000, + 0x36a: 0x3835, 0x36b: 0x383b, 0x36c: 0x378d, 0x36d: 0x3811, 0x36e: 0x3769, 0x36f: 0x37ed, + 0x370: 0x3775, 0x371: 0x37f9, 0x372: 0x377b, 0x373: 0x37ff, 0x374: 0x3781, 0x375: 0x3805, + 0x378: 0x3787, 0x379: 0x380b, + // Block 0xe, offset 0x380 + 0x387: 0x1d61, + 0x391: 0x812d, + 0x392: 0x8132, 0x393: 0x8132, 0x394: 0x8132, 0x395: 0x8132, 0x396: 0x812d, 0x397: 0x8132, + 0x398: 0x8132, 0x399: 0x8132, 0x39a: 0x812e, 0x39b: 0x812d, 0x39c: 0x8132, 0x39d: 0x8132, + 0x39e: 0x8132, 0x39f: 0x8132, 0x3a0: 0x8132, 0x3a1: 0x8132, 0x3a2: 0x812d, 0x3a3: 0x812d, + 0x3a4: 0x812d, 0x3a5: 0x812d, 0x3a6: 0x812d, 0x3a7: 0x812d, 0x3a8: 0x8132, 0x3a9: 0x8132, + 0x3aa: 0x812d, 0x3ab: 0x8132, 0x3ac: 0x8132, 0x3ad: 0x812e, 0x3ae: 0x8131, 0x3af: 0x8132, + 0x3b0: 0x8105, 0x3b1: 0x8106, 0x3b2: 0x8107, 0x3b3: 0x8108, 0x3b4: 0x8109, 0x3b5: 0x810a, + 0x3b6: 0x810b, 0x3b7: 0x810c, 0x3b8: 0x810d, 0x3b9: 0x810e, 0x3ba: 0x810e, 0x3bb: 0x810f, + 0x3bc: 0x8110, 0x3bd: 0x8111, 0x3bf: 0x8112, + // Block 0xf, offset 0x3c0 + 0x3c8: 0xa000, 0x3ca: 0xa000, 0x3cb: 0x8116, + 0x3cc: 0x8117, 0x3cd: 0x8118, 0x3ce: 0x8119, 0x3cf: 0x811a, 0x3d0: 0x811b, 0x3d1: 0x811c, + 0x3d2: 0x811d, 0x3d3: 0x9932, 0x3d4: 0x9932, 0x3d5: 0x992d, 0x3d6: 0x812d, 0x3d7: 0x8132, + 0x3d8: 0x8132, 0x3d9: 0x8132, 0x3da: 0x8132, 0x3db: 0x8132, 0x3dc: 0x812d, 0x3dd: 0x8132, + 0x3de: 0x8132, 0x3df: 0x812d, + 0x3f0: 0x811e, 0x3f5: 0x1d84, + 0x3f6: 0x2013, 0x3f7: 0x204f, 0x3f8: 0x204a, + // Block 0x10, offset 0x400 + 0x405: 0xa000, + 0x406: 0x2d26, 0x407: 0xa000, 0x408: 0x2d2e, 0x409: 0xa000, 0x40a: 0x2d36, 0x40b: 0xa000, + 0x40c: 0x2d3e, 0x40d: 0xa000, 0x40e: 0x2d46, 0x411: 0xa000, + 0x412: 0x2d4e, + 0x434: 0x8102, 0x435: 0x9900, + 0x43a: 0xa000, 0x43b: 0x2d56, + 0x43c: 0xa000, 0x43d: 0x2d5e, 0x43e: 0xa000, 0x43f: 0xa000, + // Block 0x11, offset 0x440 + 0x440: 0x0069, 0x441: 0x006b, 0x442: 0x006f, 0x443: 0x0083, 0x444: 0x00f5, 0x445: 0x00f8, + 0x446: 0x0413, 0x447: 0x0085, 0x448: 0x0089, 0x449: 0x008b, 0x44a: 0x0104, 0x44b: 0x0107, + 0x44c: 0x010a, 0x44d: 0x008f, 0x44f: 0x0097, 0x450: 0x009b, 0x451: 0x00e0, + 0x452: 0x009f, 0x453: 0x00fe, 0x454: 0x0417, 0x455: 0x041b, 0x456: 0x00a1, 0x457: 0x00a9, + 0x458: 0x00ab, 0x459: 0x0423, 0x45a: 0x012b, 0x45b: 0x00ad, 0x45c: 0x0427, 0x45d: 0x01be, + 0x45e: 0x01c1, 0x45f: 0x01c4, 0x460: 0x01fa, 0x461: 0x01fd, 0x462: 0x0093, 0x463: 0x00a5, + 0x464: 0x00ab, 0x465: 0x00ad, 0x466: 0x01be, 0x467: 0x01c1, 0x468: 0x01eb, 0x469: 0x01fa, + 0x46a: 0x01fd, + 0x478: 0x020c, + // Block 0x12, offset 0x480 + 0x49b: 0x00fb, 0x49c: 0x0087, 0x49d: 0x0101, + 0x49e: 0x00d4, 0x49f: 0x010a, 0x4a0: 0x008d, 0x4a1: 0x010d, 0x4a2: 0x0110, 0x4a3: 0x0116, + 0x4a4: 0x011c, 0x4a5: 0x011f, 0x4a6: 0x0122, 0x4a7: 0x042b, 0x4a8: 0x016a, 0x4a9: 0x0128, + 0x4aa: 0x042f, 0x4ab: 0x016d, 0x4ac: 0x0131, 0x4ad: 0x012e, 0x4ae: 0x0134, 0x4af: 0x0137, + 0x4b0: 0x013a, 0x4b1: 0x013d, 0x4b2: 0x0140, 0x4b3: 0x014c, 0x4b4: 0x014f, 0x4b5: 0x00ec, + 0x4b6: 0x0152, 0x4b7: 0x0155, 0x4b8: 0x041f, 0x4b9: 0x0158, 0x4ba: 0x015b, 0x4bb: 0x00b5, + 0x4bc: 0x015e, 0x4bd: 0x0161, 0x4be: 0x0164, 0x4bf: 0x01d0, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x2f97, 0x4c1: 0x32a3, 0x4c2: 0x2fa1, 0x4c3: 0x32ad, 0x4c4: 0x2fa6, 0x4c5: 0x32b2, + 0x4c6: 0x2fab, 0x4c7: 0x32b7, 0x4c8: 0x38cc, 0x4c9: 0x3a5b, 0x4ca: 0x2fc4, 0x4cb: 0x32d0, + 0x4cc: 0x2fce, 0x4cd: 0x32da, 0x4ce: 0x2fdd, 0x4cf: 0x32e9, 0x4d0: 0x2fd3, 0x4d1: 0x32df, + 0x4d2: 0x2fd8, 0x4d3: 0x32e4, 0x4d4: 0x38ef, 0x4d5: 0x3a7e, 0x4d6: 0x38f6, 0x4d7: 0x3a85, + 0x4d8: 0x3019, 0x4d9: 0x3325, 0x4da: 0x301e, 0x4db: 0x332a, 0x4dc: 0x3904, 0x4dd: 0x3a93, + 0x4de: 0x3023, 0x4df: 0x332f, 0x4e0: 0x3032, 0x4e1: 0x333e, 0x4e2: 0x3050, 0x4e3: 0x335c, + 0x4e4: 0x305f, 0x4e5: 0x336b, 0x4e6: 0x3055, 0x4e7: 0x3361, 0x4e8: 0x3064, 0x4e9: 0x3370, + 0x4ea: 0x3069, 0x4eb: 0x3375, 0x4ec: 0x30af, 0x4ed: 0x33bb, 0x4ee: 0x390b, 0x4ef: 0x3a9a, + 0x4f0: 0x30b9, 0x4f1: 0x33ca, 0x4f2: 0x30c3, 0x4f3: 0x33d4, 0x4f4: 0x30cd, 0x4f5: 0x33de, + 0x4f6: 0x475a, 0x4f7: 0x47eb, 0x4f8: 0x3912, 0x4f9: 0x3aa1, 0x4fa: 0x30e6, 0x4fb: 0x33f7, + 0x4fc: 0x30e1, 0x4fd: 0x33f2, 0x4fe: 0x30eb, 0x4ff: 0x33fc, + // Block 0x14, offset 0x500 + 0x500: 0x30f0, 0x501: 0x3401, 0x502: 0x30f5, 0x503: 0x3406, 0x504: 0x3109, 0x505: 0x341a, + 0x506: 0x3113, 0x507: 0x3424, 0x508: 0x3122, 0x509: 0x3433, 0x50a: 0x311d, 0x50b: 0x342e, + 0x50c: 0x3935, 0x50d: 0x3ac4, 0x50e: 0x3943, 0x50f: 0x3ad2, 0x510: 0x394a, 0x511: 0x3ad9, + 0x512: 0x3951, 0x513: 0x3ae0, 0x514: 0x314f, 0x515: 0x3460, 0x516: 0x3154, 0x517: 0x3465, + 0x518: 0x315e, 0x519: 0x346f, 0x51a: 0x4787, 0x51b: 0x4818, 0x51c: 0x3997, 0x51d: 0x3b26, + 0x51e: 0x3177, 0x51f: 0x3488, 0x520: 0x3181, 0x521: 0x3492, 0x522: 0x4796, 0x523: 0x4827, + 0x524: 0x399e, 0x525: 0x3b2d, 0x526: 0x39a5, 0x527: 0x3b34, 0x528: 0x39ac, 0x529: 0x3b3b, + 0x52a: 0x3190, 0x52b: 0x34a1, 0x52c: 0x319a, 0x52d: 0x34b0, 0x52e: 0x31ae, 0x52f: 0x34c4, + 0x530: 0x31a9, 0x531: 0x34bf, 0x532: 0x31ea, 0x533: 0x3500, 0x534: 0x31f9, 0x535: 0x350f, + 0x536: 0x31f4, 0x537: 0x350a, 0x538: 0x39b3, 0x539: 0x3b42, 0x53a: 0x39ba, 0x53b: 0x3b49, + 0x53c: 0x31fe, 0x53d: 0x3514, 0x53e: 0x3203, 0x53f: 0x3519, + // Block 0x15, offset 0x540 + 0x540: 0x3208, 0x541: 0x351e, 0x542: 0x320d, 0x543: 0x3523, 0x544: 0x321c, 0x545: 0x3532, + 0x546: 0x3217, 0x547: 0x352d, 0x548: 0x3221, 0x549: 0x353c, 0x54a: 0x3226, 0x54b: 0x3541, + 0x54c: 0x322b, 0x54d: 0x3546, 0x54e: 0x3249, 0x54f: 0x3564, 0x550: 0x3262, 0x551: 0x3582, + 0x552: 0x3271, 0x553: 0x3591, 0x554: 0x3276, 0x555: 0x3596, 0x556: 0x337a, 0x557: 0x34a6, + 0x558: 0x3537, 0x559: 0x3573, 0x55a: 0x1be0, 0x55b: 0x42d7, + 0x560: 0x4737, 0x561: 0x47c8, 0x562: 0x2f83, 0x563: 0x328f, + 0x564: 0x3878, 0x565: 0x3a07, 0x566: 0x3871, 0x567: 0x3a00, 0x568: 0x3886, 0x569: 0x3a15, + 0x56a: 0x387f, 0x56b: 0x3a0e, 0x56c: 0x38be, 0x56d: 0x3a4d, 0x56e: 0x3894, 0x56f: 0x3a23, + 0x570: 0x388d, 0x571: 0x3a1c, 0x572: 0x38a2, 0x573: 0x3a31, 0x574: 0x389b, 0x575: 0x3a2a, + 0x576: 0x38c5, 0x577: 0x3a54, 0x578: 0x474b, 0x579: 0x47dc, 0x57a: 0x3000, 0x57b: 0x330c, + 0x57c: 0x2fec, 0x57d: 0x32f8, 0x57e: 0x38da, 0x57f: 0x3a69, + // Block 0x16, offset 0x580 + 0x580: 0x38d3, 0x581: 0x3a62, 0x582: 0x38e8, 0x583: 0x3a77, 0x584: 0x38e1, 0x585: 0x3a70, + 0x586: 0x38fd, 0x587: 0x3a8c, 0x588: 0x3091, 0x589: 0x339d, 0x58a: 0x30a5, 0x58b: 0x33b1, + 0x58c: 0x477d, 0x58d: 0x480e, 0x58e: 0x3136, 0x58f: 0x3447, 0x590: 0x3920, 0x591: 0x3aaf, + 0x592: 0x3919, 0x593: 0x3aa8, 0x594: 0x392e, 0x595: 0x3abd, 0x596: 0x3927, 0x597: 0x3ab6, + 0x598: 0x3989, 0x599: 0x3b18, 0x59a: 0x396d, 0x59b: 0x3afc, 0x59c: 0x3966, 0x59d: 0x3af5, + 0x59e: 0x397b, 0x59f: 0x3b0a, 0x5a0: 0x3974, 0x5a1: 0x3b03, 0x5a2: 0x3982, 0x5a3: 0x3b11, + 0x5a4: 0x31e5, 0x5a5: 0x34fb, 0x5a6: 0x31c7, 0x5a7: 0x34dd, 0x5a8: 0x39e4, 0x5a9: 0x3b73, + 0x5aa: 0x39dd, 0x5ab: 0x3b6c, 0x5ac: 0x39f2, 0x5ad: 0x3b81, 0x5ae: 0x39eb, 0x5af: 0x3b7a, + 0x5b0: 0x39f9, 0x5b1: 0x3b88, 0x5b2: 0x3230, 0x5b3: 0x354b, 0x5b4: 0x3258, 0x5b5: 0x3578, + 0x5b6: 0x3253, 0x5b7: 0x356e, 0x5b8: 0x323f, 0x5b9: 0x355a, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x489a, 0x5c1: 0x48a0, 0x5c2: 0x49b4, 0x5c3: 0x49cc, 0x5c4: 0x49bc, 0x5c5: 0x49d4, + 0x5c6: 0x49c4, 0x5c7: 0x49dc, 0x5c8: 0x4840, 0x5c9: 0x4846, 0x5ca: 0x4924, 0x5cb: 0x493c, + 0x5cc: 0x492c, 0x5cd: 0x4944, 0x5ce: 0x4934, 0x5cf: 0x494c, 0x5d0: 0x48ac, 0x5d1: 0x48b2, + 0x5d2: 0x3db8, 0x5d3: 0x3dc8, 0x5d4: 0x3dc0, 0x5d5: 0x3dd0, + 0x5d8: 0x484c, 0x5d9: 0x4852, 0x5da: 0x3ce8, 0x5db: 0x3cf8, 0x5dc: 0x3cf0, 0x5dd: 0x3d00, + 0x5e0: 0x48c4, 0x5e1: 0x48ca, 0x5e2: 0x49e4, 0x5e3: 0x49fc, + 0x5e4: 0x49ec, 0x5e5: 0x4a04, 0x5e6: 0x49f4, 0x5e7: 0x4a0c, 0x5e8: 0x4858, 0x5e9: 0x485e, + 0x5ea: 0x4954, 0x5eb: 0x496c, 0x5ec: 0x495c, 0x5ed: 0x4974, 0x5ee: 0x4964, 0x5ef: 0x497c, + 0x5f0: 0x48dc, 0x5f1: 0x48e2, 0x5f2: 0x3e18, 0x5f3: 0x3e30, 0x5f4: 0x3e20, 0x5f5: 0x3e38, + 0x5f6: 0x3e28, 0x5f7: 0x3e40, 0x5f8: 0x4864, 0x5f9: 0x486a, 0x5fa: 0x3d18, 0x5fb: 0x3d30, + 0x5fc: 0x3d20, 0x5fd: 0x3d38, 0x5fe: 0x3d28, 0x5ff: 0x3d40, + // Block 0x18, offset 0x600 + 0x600: 0x48e8, 0x601: 0x48ee, 0x602: 0x3e48, 0x603: 0x3e58, 0x604: 0x3e50, 0x605: 0x3e60, + 0x608: 0x4870, 0x609: 0x4876, 0x60a: 0x3d48, 0x60b: 0x3d58, + 0x60c: 0x3d50, 0x60d: 0x3d60, 0x610: 0x48fa, 0x611: 0x4900, + 0x612: 0x3e80, 0x613: 0x3e98, 0x614: 0x3e88, 0x615: 0x3ea0, 0x616: 0x3e90, 0x617: 0x3ea8, + 0x619: 0x487c, 0x61b: 0x3d68, 0x61d: 0x3d70, + 0x61f: 0x3d78, 0x620: 0x4912, 0x621: 0x4918, 0x622: 0x4a14, 0x623: 0x4a2c, + 0x624: 0x4a1c, 0x625: 0x4a34, 0x626: 0x4a24, 0x627: 0x4a3c, 0x628: 0x4882, 0x629: 0x4888, + 0x62a: 0x4984, 0x62b: 0x499c, 0x62c: 0x498c, 0x62d: 0x49a4, 0x62e: 0x4994, 0x62f: 0x49ac, + 0x630: 0x488e, 0x631: 0x43b4, 0x632: 0x3691, 0x633: 0x43ba, 0x634: 0x48b8, 0x635: 0x43c0, + 0x636: 0x36a3, 0x637: 0x43c6, 0x638: 0x36c1, 0x639: 0x43cc, 0x63a: 0x36d9, 0x63b: 0x43d2, + 0x63c: 0x4906, 0x63d: 0x43d8, + // Block 0x19, offset 0x640 + 0x640: 0x3da0, 0x641: 0x3da8, 0x642: 0x4184, 0x643: 0x41a2, 0x644: 0x418e, 0x645: 0x41ac, + 0x646: 0x4198, 0x647: 0x41b6, 0x648: 0x3cd8, 0x649: 0x3ce0, 0x64a: 0x40d0, 0x64b: 0x40ee, + 0x64c: 0x40da, 0x64d: 0x40f8, 0x64e: 0x40e4, 0x64f: 0x4102, 0x650: 0x3de8, 0x651: 0x3df0, + 0x652: 0x41c0, 0x653: 0x41de, 0x654: 0x41ca, 0x655: 0x41e8, 0x656: 0x41d4, 0x657: 0x41f2, + 0x658: 0x3d08, 0x659: 0x3d10, 0x65a: 0x410c, 0x65b: 0x412a, 0x65c: 0x4116, 0x65d: 0x4134, + 0x65e: 0x4120, 0x65f: 0x413e, 0x660: 0x3ec0, 0x661: 0x3ec8, 0x662: 0x41fc, 0x663: 0x421a, + 0x664: 0x4206, 0x665: 0x4224, 0x666: 0x4210, 0x667: 0x422e, 0x668: 0x3d80, 0x669: 0x3d88, + 0x66a: 0x4148, 0x66b: 0x4166, 0x66c: 0x4152, 0x66d: 0x4170, 0x66e: 0x415c, 0x66f: 0x417a, + 0x670: 0x3685, 0x671: 0x367f, 0x672: 0x3d90, 0x673: 0x368b, 0x674: 0x3d98, + 0x676: 0x48a6, 0x677: 0x3db0, 0x678: 0x35f5, 0x679: 0x35ef, 0x67a: 0x35e3, 0x67b: 0x4384, + 0x67c: 0x35fb, 0x67d: 0x4287, 0x67e: 0x01d3, 0x67f: 0x4287, + // Block 0x1a, offset 0x680 + 0x680: 0x42a0, 0x681: 0x4518, 0x682: 0x3dd8, 0x683: 0x369d, 0x684: 0x3de0, + 0x686: 0x48d0, 0x687: 0x3df8, 0x688: 0x3601, 0x689: 0x438a, 0x68a: 0x360d, 0x68b: 0x4390, + 0x68c: 0x3619, 0x68d: 0x451f, 0x68e: 0x4526, 0x68f: 0x452d, 0x690: 0x36b5, 0x691: 0x36af, + 0x692: 0x3e00, 0x693: 0x457a, 0x696: 0x36bb, 0x697: 0x3e10, + 0x698: 0x3631, 0x699: 0x362b, 0x69a: 0x361f, 0x69b: 0x4396, 0x69d: 0x4534, + 0x69e: 0x453b, 0x69f: 0x4542, 0x6a0: 0x36eb, 0x6a1: 0x36e5, 0x6a2: 0x3e68, 0x6a3: 0x4582, + 0x6a4: 0x36cd, 0x6a5: 0x36d3, 0x6a6: 0x36f1, 0x6a7: 0x3e78, 0x6a8: 0x3661, 0x6a9: 0x365b, + 0x6aa: 0x364f, 0x6ab: 0x43a2, 0x6ac: 0x3649, 0x6ad: 0x450a, 0x6ae: 0x4511, 0x6af: 0x0081, + 0x6b2: 0x3eb0, 0x6b3: 0x36f7, 0x6b4: 0x3eb8, + 0x6b6: 0x491e, 0x6b7: 0x3ed0, 0x6b8: 0x363d, 0x6b9: 0x439c, 0x6ba: 0x366d, 0x6bb: 0x43ae, + 0x6bc: 0x3679, 0x6bd: 0x425a, 0x6be: 0x428c, + // Block 0x1b, offset 0x6c0 + 0x6c0: 0x1bd8, 0x6c1: 0x1bdc, 0x6c2: 0x0047, 0x6c3: 0x1c54, 0x6c5: 0x1be8, + 0x6c6: 0x1bec, 0x6c7: 0x00e9, 0x6c9: 0x1c58, 0x6ca: 0x008f, 0x6cb: 0x0051, + 0x6cc: 0x0051, 0x6cd: 0x0051, 0x6ce: 0x0091, 0x6cf: 0x00da, 0x6d0: 0x0053, 0x6d1: 0x0053, + 0x6d2: 0x0059, 0x6d3: 0x0099, 0x6d5: 0x005d, 0x6d6: 0x198d, + 0x6d9: 0x0061, 0x6da: 0x0063, 0x6db: 0x0065, 0x6dc: 0x0065, 0x6dd: 0x0065, + 0x6e0: 0x199f, 0x6e1: 0x1bc8, 0x6e2: 0x19a8, + 0x6e4: 0x0075, 0x6e6: 0x01b8, 0x6e8: 0x0075, + 0x6ea: 0x0057, 0x6eb: 0x42d2, 0x6ec: 0x0045, 0x6ed: 0x0047, 0x6ef: 0x008b, + 0x6f0: 0x004b, 0x6f1: 0x004d, 0x6f3: 0x005b, 0x6f4: 0x009f, 0x6f5: 0x0215, + 0x6f6: 0x0218, 0x6f7: 0x021b, 0x6f8: 0x021e, 0x6f9: 0x0093, 0x6fb: 0x1b98, + 0x6fc: 0x01e8, 0x6fd: 0x01c1, 0x6fe: 0x0179, 0x6ff: 0x01a0, + // Block 0x1c, offset 0x700 + 0x700: 0x0463, 0x705: 0x0049, + 0x706: 0x0089, 0x707: 0x008b, 0x708: 0x0093, 0x709: 0x0095, + 0x710: 0x222e, 0x711: 0x223a, + 0x712: 0x22ee, 0x713: 0x2216, 0x714: 0x229a, 0x715: 0x2222, 0x716: 0x22a0, 0x717: 0x22b8, + 0x718: 0x22c4, 0x719: 0x2228, 0x71a: 0x22ca, 0x71b: 0x2234, 0x71c: 0x22be, 0x71d: 0x22d0, + 0x71e: 0x22d6, 0x71f: 0x1cbc, 0x720: 0x0053, 0x721: 0x195a, 0x722: 0x1ba4, 0x723: 0x1963, + 0x724: 0x006d, 0x725: 0x19ab, 0x726: 0x1bd0, 0x727: 0x1d48, 0x728: 0x1966, 0x729: 0x0071, + 0x72a: 0x19b7, 0x72b: 0x1bd4, 0x72c: 0x0059, 0x72d: 0x0047, 0x72e: 0x0049, 0x72f: 0x005b, + 0x730: 0x0093, 0x731: 0x19e4, 0x732: 0x1c18, 0x733: 0x19ed, 0x734: 0x00ad, 0x735: 0x1a62, + 0x736: 0x1c4c, 0x737: 0x1d5c, 0x738: 0x19f0, 0x739: 0x00b1, 0x73a: 0x1a65, 0x73b: 0x1c50, + 0x73c: 0x0099, 0x73d: 0x0087, 0x73e: 0x0089, 0x73f: 0x009b, + // Block 0x1d, offset 0x740 + 0x741: 0x3c06, 0x743: 0xa000, 0x744: 0x3c0d, 0x745: 0xa000, + 0x747: 0x3c14, 0x748: 0xa000, 0x749: 0x3c1b, + 0x74d: 0xa000, + 0x760: 0x2f65, 0x761: 0xa000, 0x762: 0x3c29, + 0x764: 0xa000, 0x765: 0xa000, + 0x76d: 0x3c22, 0x76e: 0x2f60, 0x76f: 0x2f6a, + 0x770: 0x3c30, 0x771: 0x3c37, 0x772: 0xa000, 0x773: 0xa000, 0x774: 0x3c3e, 0x775: 0x3c45, + 0x776: 0xa000, 0x777: 0xa000, 0x778: 0x3c4c, 0x779: 0x3c53, 0x77a: 0xa000, 0x77b: 0xa000, + 0x77c: 0xa000, 0x77d: 0xa000, + // Block 0x1e, offset 0x780 + 0x780: 0x3c5a, 0x781: 0x3c61, 0x782: 0xa000, 0x783: 0xa000, 0x784: 0x3c76, 0x785: 0x3c7d, + 0x786: 0xa000, 0x787: 0xa000, 0x788: 0x3c84, 0x789: 0x3c8b, + 0x791: 0xa000, + 0x792: 0xa000, + 0x7a2: 0xa000, + 0x7a8: 0xa000, 0x7a9: 0xa000, + 0x7ab: 0xa000, 0x7ac: 0x3ca0, 0x7ad: 0x3ca7, 0x7ae: 0x3cae, 0x7af: 0x3cb5, + 0x7b2: 0xa000, 0x7b3: 0xa000, 0x7b4: 0xa000, 0x7b5: 0xa000, + // Block 0x1f, offset 0x7c0 + 0x7e0: 0x0023, 0x7e1: 0x0025, 0x7e2: 0x0027, 0x7e3: 0x0029, + 0x7e4: 0x002b, 0x7e5: 0x002d, 0x7e6: 0x002f, 0x7e7: 0x0031, 0x7e8: 0x0033, 0x7e9: 0x1882, + 0x7ea: 0x1885, 0x7eb: 0x1888, 0x7ec: 0x188b, 0x7ed: 0x188e, 0x7ee: 0x1891, 0x7ef: 0x1894, + 0x7f0: 0x1897, 0x7f1: 0x189a, 0x7f2: 0x189d, 0x7f3: 0x18a6, 0x7f4: 0x1a68, 0x7f5: 0x1a6c, + 0x7f6: 0x1a70, 0x7f7: 0x1a74, 0x7f8: 0x1a78, 0x7f9: 0x1a7c, 0x7fa: 0x1a80, 0x7fb: 0x1a84, + 0x7fc: 0x1a88, 0x7fd: 0x1c80, 0x7fe: 0x1c85, 0x7ff: 0x1c8a, + // Block 0x20, offset 0x800 + 0x800: 0x1c8f, 0x801: 0x1c94, 0x802: 0x1c99, 0x803: 0x1c9e, 0x804: 0x1ca3, 0x805: 0x1ca8, + 0x806: 0x1cad, 0x807: 0x1cb2, 0x808: 0x187f, 0x809: 0x18a3, 0x80a: 0x18c7, 0x80b: 0x18eb, + 0x80c: 0x190f, 0x80d: 0x1918, 0x80e: 0x191e, 0x80f: 0x1924, 0x810: 0x192a, 0x811: 0x1b60, + 0x812: 0x1b64, 0x813: 0x1b68, 0x814: 0x1b6c, 0x815: 0x1b70, 0x816: 0x1b74, 0x817: 0x1b78, + 0x818: 0x1b7c, 0x819: 0x1b80, 0x81a: 0x1b84, 0x81b: 0x1b88, 0x81c: 0x1af4, 0x81d: 0x1af8, + 0x81e: 0x1afc, 0x81f: 0x1b00, 0x820: 0x1b04, 0x821: 0x1b08, 0x822: 0x1b0c, 0x823: 0x1b10, + 0x824: 0x1b14, 0x825: 0x1b18, 0x826: 0x1b1c, 0x827: 0x1b20, 0x828: 0x1b24, 0x829: 0x1b28, + 0x82a: 0x1b2c, 0x82b: 0x1b30, 0x82c: 0x1b34, 0x82d: 0x1b38, 0x82e: 0x1b3c, 0x82f: 0x1b40, + 0x830: 0x1b44, 0x831: 0x1b48, 0x832: 0x1b4c, 0x833: 0x1b50, 0x834: 0x1b54, 0x835: 0x1b58, + 0x836: 0x0043, 0x837: 0x0045, 0x838: 0x0047, 0x839: 0x0049, 0x83a: 0x004b, 0x83b: 0x004d, + 0x83c: 0x004f, 0x83d: 0x0051, 0x83e: 0x0053, 0x83f: 0x0055, + // Block 0x21, offset 0x840 + 0x840: 0x06bf, 0x841: 0x06e3, 0x842: 0x06ef, 0x843: 0x06ff, 0x844: 0x0707, 0x845: 0x0713, + 0x846: 0x071b, 0x847: 0x0723, 0x848: 0x072f, 0x849: 0x0783, 0x84a: 0x079b, 0x84b: 0x07ab, + 0x84c: 0x07bb, 0x84d: 0x07cb, 0x84e: 0x07db, 0x84f: 0x07fb, 0x850: 0x07ff, 0x851: 0x0803, + 0x852: 0x0837, 0x853: 0x085f, 0x854: 0x086f, 0x855: 0x0877, 0x856: 0x087b, 0x857: 0x0887, + 0x858: 0x08a3, 0x859: 0x08a7, 0x85a: 0x08bf, 0x85b: 0x08c3, 0x85c: 0x08cb, 0x85d: 0x08db, + 0x85e: 0x0977, 0x85f: 0x098b, 0x860: 0x09cb, 0x861: 0x09df, 0x862: 0x09e7, 0x863: 0x09eb, + 0x864: 0x09fb, 0x865: 0x0a17, 0x866: 0x0a43, 0x867: 0x0a4f, 0x868: 0x0a6f, 0x869: 0x0a7b, + 0x86a: 0x0a7f, 0x86b: 0x0a83, 0x86c: 0x0a9b, 0x86d: 0x0a9f, 0x86e: 0x0acb, 0x86f: 0x0ad7, + 0x870: 0x0adf, 0x871: 0x0ae7, 0x872: 0x0af7, 0x873: 0x0aff, 0x874: 0x0b07, 0x875: 0x0b33, + 0x876: 0x0b37, 0x877: 0x0b3f, 0x878: 0x0b43, 0x879: 0x0b4b, 0x87a: 0x0b53, 0x87b: 0x0b63, + 0x87c: 0x0b7f, 0x87d: 0x0bf7, 0x87e: 0x0c0b, 0x87f: 0x0c0f, + // Block 0x22, offset 0x880 + 0x880: 0x0c8f, 0x881: 0x0c93, 0x882: 0x0ca7, 0x883: 0x0cab, 0x884: 0x0cb3, 0x885: 0x0cbb, + 0x886: 0x0cc3, 0x887: 0x0ccf, 0x888: 0x0cf7, 0x889: 0x0d07, 0x88a: 0x0d1b, 0x88b: 0x0d8b, + 0x88c: 0x0d97, 0x88d: 0x0da7, 0x88e: 0x0db3, 0x88f: 0x0dbf, 0x890: 0x0dc7, 0x891: 0x0dcb, + 0x892: 0x0dcf, 0x893: 0x0dd3, 0x894: 0x0dd7, 0x895: 0x0e8f, 0x896: 0x0ed7, 0x897: 0x0ee3, + 0x898: 0x0ee7, 0x899: 0x0eeb, 0x89a: 0x0eef, 0x89b: 0x0ef7, 0x89c: 0x0efb, 0x89d: 0x0f0f, + 0x89e: 0x0f2b, 0x89f: 0x0f33, 0x8a0: 0x0f73, 0x8a1: 0x0f77, 0x8a2: 0x0f7f, 0x8a3: 0x0f83, + 0x8a4: 0x0f8b, 0x8a5: 0x0f8f, 0x8a6: 0x0fb3, 0x8a7: 0x0fb7, 0x8a8: 0x0fd3, 0x8a9: 0x0fd7, + 0x8aa: 0x0fdb, 0x8ab: 0x0fdf, 0x8ac: 0x0ff3, 0x8ad: 0x1017, 0x8ae: 0x101b, 0x8af: 0x101f, + 0x8b0: 0x1043, 0x8b1: 0x1083, 0x8b2: 0x1087, 0x8b3: 0x10a7, 0x8b4: 0x10b7, 0x8b5: 0x10bf, + 0x8b6: 0x10df, 0x8b7: 0x1103, 0x8b8: 0x1147, 0x8b9: 0x114f, 0x8ba: 0x1163, 0x8bb: 0x116f, + 0x8bc: 0x1177, 0x8bd: 0x117f, 0x8be: 0x1183, 0x8bf: 0x1187, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x119f, 0x8c1: 0x11a3, 0x8c2: 0x11bf, 0x8c3: 0x11c7, 0x8c4: 0x11cf, 0x8c5: 0x11d3, + 0x8c6: 0x11df, 0x8c7: 0x11e7, 0x8c8: 0x11eb, 0x8c9: 0x11ef, 0x8ca: 0x11f7, 0x8cb: 0x11fb, + 0x8cc: 0x129b, 0x8cd: 0x12af, 0x8ce: 0x12e3, 0x8cf: 0x12e7, 0x8d0: 0x12ef, 0x8d1: 0x131b, + 0x8d2: 0x1323, 0x8d3: 0x132b, 0x8d4: 0x1333, 0x8d5: 0x136f, 0x8d6: 0x1373, 0x8d7: 0x137b, + 0x8d8: 0x137f, 0x8d9: 0x1383, 0x8da: 0x13af, 0x8db: 0x13b3, 0x8dc: 0x13bb, 0x8dd: 0x13cf, + 0x8de: 0x13d3, 0x8df: 0x13ef, 0x8e0: 0x13f7, 0x8e1: 0x13fb, 0x8e2: 0x141f, 0x8e3: 0x143f, + 0x8e4: 0x1453, 0x8e5: 0x1457, 0x8e6: 0x145f, 0x8e7: 0x148b, 0x8e8: 0x148f, 0x8e9: 0x149f, + 0x8ea: 0x14c3, 0x8eb: 0x14cf, 0x8ec: 0x14df, 0x8ed: 0x14f7, 0x8ee: 0x14ff, 0x8ef: 0x1503, + 0x8f0: 0x1507, 0x8f1: 0x150b, 0x8f2: 0x1517, 0x8f3: 0x151b, 0x8f4: 0x1523, 0x8f5: 0x153f, + 0x8f6: 0x1543, 0x8f7: 0x1547, 0x8f8: 0x155f, 0x8f9: 0x1563, 0x8fa: 0x156b, 0x8fb: 0x157f, + 0x8fc: 0x1583, 0x8fd: 0x1587, 0x8fe: 0x158f, 0x8ff: 0x1593, + // Block 0x24, offset 0x900 + 0x906: 0xa000, 0x90b: 0xa000, + 0x90c: 0x3f08, 0x90d: 0xa000, 0x90e: 0x3f10, 0x90f: 0xa000, 0x910: 0x3f18, 0x911: 0xa000, + 0x912: 0x3f20, 0x913: 0xa000, 0x914: 0x3f28, 0x915: 0xa000, 0x916: 0x3f30, 0x917: 0xa000, + 0x918: 0x3f38, 0x919: 0xa000, 0x91a: 0x3f40, 0x91b: 0xa000, 0x91c: 0x3f48, 0x91d: 0xa000, + 0x91e: 0x3f50, 0x91f: 0xa000, 0x920: 0x3f58, 0x921: 0xa000, 0x922: 0x3f60, + 0x924: 0xa000, 0x925: 0x3f68, 0x926: 0xa000, 0x927: 0x3f70, 0x928: 0xa000, 0x929: 0x3f78, + 0x92f: 0xa000, + 0x930: 0x3f80, 0x931: 0x3f88, 0x932: 0xa000, 0x933: 0x3f90, 0x934: 0x3f98, 0x935: 0xa000, + 0x936: 0x3fa0, 0x937: 0x3fa8, 0x938: 0xa000, 0x939: 0x3fb0, 0x93a: 0x3fb8, 0x93b: 0xa000, + 0x93c: 0x3fc0, 0x93d: 0x3fc8, + // Block 0x25, offset 0x940 + 0x954: 0x3f00, + 0x959: 0x9903, 0x95a: 0x9903, 0x95b: 0x4372, 0x95c: 0x4378, 0x95d: 0xa000, + 0x95e: 0x3fd0, 0x95f: 0x26b4, + 0x966: 0xa000, + 0x96b: 0xa000, 0x96c: 0x3fe0, 0x96d: 0xa000, 0x96e: 0x3fe8, 0x96f: 0xa000, + 0x970: 0x3ff0, 0x971: 0xa000, 0x972: 0x3ff8, 0x973: 0xa000, 0x974: 0x4000, 0x975: 0xa000, + 0x976: 0x4008, 0x977: 0xa000, 0x978: 0x4010, 0x979: 0xa000, 0x97a: 0x4018, 0x97b: 0xa000, + 0x97c: 0x4020, 0x97d: 0xa000, 0x97e: 0x4028, 0x97f: 0xa000, + // Block 0x26, offset 0x980 + 0x980: 0x4030, 0x981: 0xa000, 0x982: 0x4038, 0x984: 0xa000, 0x985: 0x4040, + 0x986: 0xa000, 0x987: 0x4048, 0x988: 0xa000, 0x989: 0x4050, + 0x98f: 0xa000, 0x990: 0x4058, 0x991: 0x4060, + 0x992: 0xa000, 0x993: 0x4068, 0x994: 0x4070, 0x995: 0xa000, 0x996: 0x4078, 0x997: 0x4080, + 0x998: 0xa000, 0x999: 0x4088, 0x99a: 0x4090, 0x99b: 0xa000, 0x99c: 0x4098, 0x99d: 0x40a0, + 0x9af: 0xa000, + 0x9b0: 0xa000, 0x9b1: 0xa000, 0x9b2: 0xa000, 0x9b4: 0x3fd8, + 0x9b7: 0x40a8, 0x9b8: 0x40b0, 0x9b9: 0x40b8, 0x9ba: 0x40c0, + 0x9bd: 0xa000, 0x9be: 0x40c8, 0x9bf: 0x26c9, + // Block 0x27, offset 0x9c0 + 0x9c0: 0x0367, 0x9c1: 0x032b, 0x9c2: 0x032f, 0x9c3: 0x0333, 0x9c4: 0x037b, 0x9c5: 0x0337, + 0x9c6: 0x033b, 0x9c7: 0x033f, 0x9c8: 0x0343, 0x9c9: 0x0347, 0x9ca: 0x034b, 0x9cb: 0x034f, + 0x9cc: 0x0353, 0x9cd: 0x0357, 0x9ce: 0x035b, 0x9cf: 0x42dc, 0x9d0: 0x42e1, 0x9d1: 0x42e6, + 0x9d2: 0x42eb, 0x9d3: 0x42f0, 0x9d4: 0x42f5, 0x9d5: 0x42fa, 0x9d6: 0x42ff, 0x9d7: 0x4304, + 0x9d8: 0x4309, 0x9d9: 0x430e, 0x9da: 0x4313, 0x9db: 0x4318, 0x9dc: 0x431d, 0x9dd: 0x4322, + 0x9de: 0x4327, 0x9df: 0x432c, 0x9e0: 0x4331, 0x9e1: 0x4336, 0x9e2: 0x433b, 0x9e3: 0x4340, + 0x9e4: 0x03c3, 0x9e5: 0x035f, 0x9e6: 0x0363, 0x9e7: 0x03e7, 0x9e8: 0x03eb, 0x9e9: 0x03ef, + 0x9ea: 0x03f3, 0x9eb: 0x03f7, 0x9ec: 0x03fb, 0x9ed: 0x03ff, 0x9ee: 0x036b, 0x9ef: 0x0403, + 0x9f0: 0x0407, 0x9f1: 0x036f, 0x9f2: 0x0373, 0x9f3: 0x0377, 0x9f4: 0x037f, 0x9f5: 0x0383, + 0x9f6: 0x0387, 0x9f7: 0x038b, 0x9f8: 0x038f, 0x9f9: 0x0393, 0x9fa: 0x0397, 0x9fb: 0x039b, + 0x9fc: 0x039f, 0x9fd: 0x03a3, 0x9fe: 0x03a7, 0x9ff: 0x03ab, + // Block 0x28, offset 0xa00 + 0xa00: 0x03af, 0xa01: 0x03b3, 0xa02: 0x040b, 0xa03: 0x040f, 0xa04: 0x03b7, 0xa05: 0x03bb, + 0xa06: 0x03bf, 0xa07: 0x03c7, 0xa08: 0x03cb, 0xa09: 0x03cf, 0xa0a: 0x03d3, 0xa0b: 0x03d7, + 0xa0c: 0x03db, 0xa0d: 0x03df, 0xa0e: 0x03e3, + 0xa12: 0x06bf, 0xa13: 0x071b, 0xa14: 0x06cb, 0xa15: 0x097b, 0xa16: 0x06cf, 0xa17: 0x06e7, + 0xa18: 0x06d3, 0xa19: 0x0f93, 0xa1a: 0x0707, 0xa1b: 0x06db, 0xa1c: 0x06c3, 0xa1d: 0x09ff, + 0xa1e: 0x098f, 0xa1f: 0x072f, + // Block 0x29, offset 0xa40 + 0xa40: 0x2054, 0xa41: 0x205a, 0xa42: 0x2060, 0xa43: 0x2066, 0xa44: 0x206c, 0xa45: 0x2072, + 0xa46: 0x2078, 0xa47: 0x207e, 0xa48: 0x2084, 0xa49: 0x208a, 0xa4a: 0x2090, 0xa4b: 0x2096, + 0xa4c: 0x209c, 0xa4d: 0x20a2, 0xa4e: 0x2726, 0xa4f: 0x272f, 0xa50: 0x2738, 0xa51: 0x2741, + 0xa52: 0x274a, 0xa53: 0x2753, 0xa54: 0x275c, 0xa55: 0x2765, 0xa56: 0x276e, 0xa57: 0x2780, + 0xa58: 0x2789, 0xa59: 0x2792, 0xa5a: 0x279b, 0xa5b: 0x27a4, 0xa5c: 0x2777, 0xa5d: 0x2bac, + 0xa5e: 0x2aed, 0xa60: 0x20a8, 0xa61: 0x20c0, 0xa62: 0x20b4, 0xa63: 0x2108, + 0xa64: 0x20c6, 0xa65: 0x20e4, 0xa66: 0x20ae, 0xa67: 0x20de, 0xa68: 0x20ba, 0xa69: 0x20f0, + 0xa6a: 0x2120, 0xa6b: 0x213e, 0xa6c: 0x2138, 0xa6d: 0x212c, 0xa6e: 0x217a, 0xa6f: 0x210e, + 0xa70: 0x211a, 0xa71: 0x2132, 0xa72: 0x2126, 0xa73: 0x2150, 0xa74: 0x20fc, 0xa75: 0x2144, + 0xa76: 0x216e, 0xa77: 0x2156, 0xa78: 0x20ea, 0xa79: 0x20cc, 0xa7a: 0x2102, 0xa7b: 0x2114, + 0xa7c: 0x214a, 0xa7d: 0x20d2, 0xa7e: 0x2174, 0xa7f: 0x20f6, + // Block 0x2a, offset 0xa80 + 0xa80: 0x215c, 0xa81: 0x20d8, 0xa82: 0x2162, 0xa83: 0x2168, 0xa84: 0x092f, 0xa85: 0x0b03, + 0xa86: 0x0ca7, 0xa87: 0x10c7, + 0xa90: 0x1bc4, 0xa91: 0x18a9, + 0xa92: 0x18ac, 0xa93: 0x18af, 0xa94: 0x18b2, 0xa95: 0x18b5, 0xa96: 0x18b8, 0xa97: 0x18bb, + 0xa98: 0x18be, 0xa99: 0x18c1, 0xa9a: 0x18ca, 0xa9b: 0x18cd, 0xa9c: 0x18d0, 0xa9d: 0x18d3, + 0xa9e: 0x18d6, 0xa9f: 0x18d9, 0xaa0: 0x0313, 0xaa1: 0x031b, 0xaa2: 0x031f, 0xaa3: 0x0327, + 0xaa4: 0x032b, 0xaa5: 0x032f, 0xaa6: 0x0337, 0xaa7: 0x033f, 0xaa8: 0x0343, 0xaa9: 0x034b, + 0xaaa: 0x034f, 0xaab: 0x0353, 0xaac: 0x0357, 0xaad: 0x035b, 0xaae: 0x2e18, 0xaaf: 0x2e20, + 0xab0: 0x2e28, 0xab1: 0x2e30, 0xab2: 0x2e38, 0xab3: 0x2e40, 0xab4: 0x2e48, 0xab5: 0x2e50, + 0xab6: 0x2e60, 0xab7: 0x2e68, 0xab8: 0x2e70, 0xab9: 0x2e78, 0xaba: 0x2e80, 0xabb: 0x2e88, + 0xabc: 0x2ed3, 0xabd: 0x2e9b, 0xabe: 0x2e58, + // Block 0x2b, offset 0xac0 + 0xac0: 0x06bf, 0xac1: 0x071b, 0xac2: 0x06cb, 0xac3: 0x097b, 0xac4: 0x071f, 0xac5: 0x07af, + 0xac6: 0x06c7, 0xac7: 0x07ab, 0xac8: 0x070b, 0xac9: 0x0887, 0xaca: 0x0d07, 0xacb: 0x0e8f, + 0xacc: 0x0dd7, 0xacd: 0x0d1b, 0xace: 0x145f, 0xacf: 0x098b, 0xad0: 0x0ccf, 0xad1: 0x0d4b, + 0xad2: 0x0d0b, 0xad3: 0x104b, 0xad4: 0x08fb, 0xad5: 0x0f03, 0xad6: 0x1387, 0xad7: 0x105f, + 0xad8: 0x0843, 0xad9: 0x108f, 0xada: 0x0f9b, 0xadb: 0x0a17, 0xadc: 0x140f, 0xadd: 0x077f, + 0xade: 0x08ab, 0xadf: 0x0df7, 0xae0: 0x1527, 0xae1: 0x0743, 0xae2: 0x07d3, 0xae3: 0x0d9b, + 0xae4: 0x06cf, 0xae5: 0x06e7, 0xae6: 0x06d3, 0xae7: 0x0adb, 0xae8: 0x08ef, 0xae9: 0x087f, + 0xaea: 0x0a57, 0xaeb: 0x0a4b, 0xaec: 0x0feb, 0xaed: 0x073f, 0xaee: 0x139b, 0xaef: 0x089b, + 0xaf0: 0x09f3, 0xaf1: 0x18dc, 0xaf2: 0x18df, 0xaf3: 0x18e2, 0xaf4: 0x18e5, 0xaf5: 0x18ee, + 0xaf6: 0x18f1, 0xaf7: 0x18f4, 0xaf8: 0x18f7, 0xaf9: 0x18fa, 0xafa: 0x18fd, 0xafb: 0x1900, + 0xafc: 0x1903, 0xafd: 0x1906, 0xafe: 0x1909, 0xaff: 0x1912, + // Block 0x2c, offset 0xb00 + 0xb00: 0x1cc6, 0xb01: 0x1cd5, 0xb02: 0x1ce4, 0xb03: 0x1cf3, 0xb04: 0x1d02, 0xb05: 0x1d11, + 0xb06: 0x1d20, 0xb07: 0x1d2f, 0xb08: 0x1d3e, 0xb09: 0x218c, 0xb0a: 0x219e, 0xb0b: 0x21b0, + 0xb0c: 0x1954, 0xb0d: 0x1c04, 0xb0e: 0x19d2, 0xb0f: 0x1ba8, 0xb10: 0x04cb, 0xb11: 0x04d3, + 0xb12: 0x04db, 0xb13: 0x04e3, 0xb14: 0x04eb, 0xb15: 0x04ef, 0xb16: 0x04f3, 0xb17: 0x04f7, + 0xb18: 0x04fb, 0xb19: 0x04ff, 0xb1a: 0x0503, 0xb1b: 0x0507, 0xb1c: 0x050b, 0xb1d: 0x050f, + 0xb1e: 0x0513, 0xb1f: 0x0517, 0xb20: 0x051b, 0xb21: 0x0523, 0xb22: 0x0527, 0xb23: 0x052b, + 0xb24: 0x052f, 0xb25: 0x0533, 0xb26: 0x0537, 0xb27: 0x053b, 0xb28: 0x053f, 0xb29: 0x0543, + 0xb2a: 0x0547, 0xb2b: 0x054b, 0xb2c: 0x054f, 0xb2d: 0x0553, 0xb2e: 0x0557, 0xb2f: 0x055b, + 0xb30: 0x055f, 0xb31: 0x0563, 0xb32: 0x0567, 0xb33: 0x056f, 0xb34: 0x0577, 0xb35: 0x057f, + 0xb36: 0x0583, 0xb37: 0x0587, 0xb38: 0x058b, 0xb39: 0x058f, 0xb3a: 0x0593, 0xb3b: 0x0597, + 0xb3c: 0x059b, 0xb3d: 0x059f, 0xb3e: 0x05a3, + // Block 0x2d, offset 0xb40 + 0xb40: 0x2b0c, 0xb41: 0x29a8, 0xb42: 0x2b1c, 0xb43: 0x2880, 0xb44: 0x2ee4, 0xb45: 0x288a, + 0xb46: 0x2894, 0xb47: 0x2f28, 0xb48: 0x29b5, 0xb49: 0x289e, 0xb4a: 0x28a8, 0xb4b: 0x28b2, + 0xb4c: 0x29dc, 0xb4d: 0x29e9, 0xb4e: 0x29c2, 0xb4f: 0x29cf, 0xb50: 0x2ea9, 0xb51: 0x29f6, + 0xb52: 0x2a03, 0xb53: 0x2bbe, 0xb54: 0x26bb, 0xb55: 0x2bd1, 0xb56: 0x2be4, 0xb57: 0x2b2c, + 0xb58: 0x2a10, 0xb59: 0x2bf7, 0xb5a: 0x2c0a, 0xb5b: 0x2a1d, 0xb5c: 0x28bc, 0xb5d: 0x28c6, + 0xb5e: 0x2eb7, 0xb5f: 0x2a2a, 0xb60: 0x2b3c, 0xb61: 0x2ef5, 0xb62: 0x28d0, 0xb63: 0x28da, + 0xb64: 0x2a37, 0xb65: 0x28e4, 0xb66: 0x28ee, 0xb67: 0x26d0, 0xb68: 0x26d7, 0xb69: 0x28f8, + 0xb6a: 0x2902, 0xb6b: 0x2c1d, 0xb6c: 0x2a44, 0xb6d: 0x2b4c, 0xb6e: 0x2c30, 0xb6f: 0x2a51, + 0xb70: 0x2916, 0xb71: 0x290c, 0xb72: 0x2f3c, 0xb73: 0x2a5e, 0xb74: 0x2c43, 0xb75: 0x2920, + 0xb76: 0x2b5c, 0xb77: 0x292a, 0xb78: 0x2a78, 0xb79: 0x2934, 0xb7a: 0x2a85, 0xb7b: 0x2f06, + 0xb7c: 0x2a6b, 0xb7d: 0x2b6c, 0xb7e: 0x2a92, 0xb7f: 0x26de, + // Block 0x2e, offset 0xb80 + 0xb80: 0x2f17, 0xb81: 0x293e, 0xb82: 0x2948, 0xb83: 0x2a9f, 0xb84: 0x2952, 0xb85: 0x295c, + 0xb86: 0x2966, 0xb87: 0x2b7c, 0xb88: 0x2aac, 0xb89: 0x26e5, 0xb8a: 0x2c56, 0xb8b: 0x2e90, + 0xb8c: 0x2b8c, 0xb8d: 0x2ab9, 0xb8e: 0x2ec5, 0xb8f: 0x2970, 0xb90: 0x297a, 0xb91: 0x2ac6, + 0xb92: 0x26ec, 0xb93: 0x2ad3, 0xb94: 0x2b9c, 0xb95: 0x26f3, 0xb96: 0x2c69, 0xb97: 0x2984, + 0xb98: 0x1cb7, 0xb99: 0x1ccb, 0xb9a: 0x1cda, 0xb9b: 0x1ce9, 0xb9c: 0x1cf8, 0xb9d: 0x1d07, + 0xb9e: 0x1d16, 0xb9f: 0x1d25, 0xba0: 0x1d34, 0xba1: 0x1d43, 0xba2: 0x2192, 0xba3: 0x21a4, + 0xba4: 0x21b6, 0xba5: 0x21c2, 0xba6: 0x21ce, 0xba7: 0x21da, 0xba8: 0x21e6, 0xba9: 0x21f2, + 0xbaa: 0x21fe, 0xbab: 0x220a, 0xbac: 0x2246, 0xbad: 0x2252, 0xbae: 0x225e, 0xbaf: 0x226a, + 0xbb0: 0x2276, 0xbb1: 0x1c14, 0xbb2: 0x19c6, 0xbb3: 0x1936, 0xbb4: 0x1be4, 0xbb5: 0x1a47, + 0xbb6: 0x1a56, 0xbb7: 0x19cc, 0xbb8: 0x1bfc, 0xbb9: 0x1c00, 0xbba: 0x1960, 0xbbb: 0x2701, + 0xbbc: 0x270f, 0xbbd: 0x26fa, 0xbbe: 0x2708, 0xbbf: 0x2ae0, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x1a4a, 0xbc1: 0x1a32, 0xbc2: 0x1c60, 0xbc3: 0x1a1a, 0xbc4: 0x19f3, 0xbc5: 0x1969, + 0xbc6: 0x1978, 0xbc7: 0x1948, 0xbc8: 0x1bf0, 0xbc9: 0x1d52, 0xbca: 0x1a4d, 0xbcb: 0x1a35, + 0xbcc: 0x1c64, 0xbcd: 0x1c70, 0xbce: 0x1a26, 0xbcf: 0x19fc, 0xbd0: 0x1957, 0xbd1: 0x1c1c, + 0xbd2: 0x1bb0, 0xbd3: 0x1b9c, 0xbd4: 0x1bcc, 0xbd5: 0x1c74, 0xbd6: 0x1a29, 0xbd7: 0x19c9, + 0xbd8: 0x19ff, 0xbd9: 0x19de, 0xbda: 0x1a41, 0xbdb: 0x1c78, 0xbdc: 0x1a2c, 0xbdd: 0x19c0, + 0xbde: 0x1a02, 0xbdf: 0x1c3c, 0xbe0: 0x1bf4, 0xbe1: 0x1a14, 0xbe2: 0x1c24, 0xbe3: 0x1c40, + 0xbe4: 0x1bf8, 0xbe5: 0x1a17, 0xbe6: 0x1c28, 0xbe7: 0x22e8, 0xbe8: 0x22fc, 0xbe9: 0x1996, + 0xbea: 0x1c20, 0xbeb: 0x1bb4, 0xbec: 0x1ba0, 0xbed: 0x1c48, 0xbee: 0x2716, 0xbef: 0x27ad, + 0xbf0: 0x1a59, 0xbf1: 0x1a44, 0xbf2: 0x1c7c, 0xbf3: 0x1a2f, 0xbf4: 0x1a50, 0xbf5: 0x1a38, + 0xbf6: 0x1c68, 0xbf7: 0x1a1d, 0xbf8: 0x19f6, 0xbf9: 0x1981, 0xbfa: 0x1a53, 0xbfb: 0x1a3b, + 0xbfc: 0x1c6c, 0xbfd: 0x1a20, 0xbfe: 0x19f9, 0xbff: 0x1984, + // Block 0x30, offset 0xc00 + 0xc00: 0x1c2c, 0xc01: 0x1bb8, 0xc02: 0x1d4d, 0xc03: 0x1939, 0xc04: 0x19ba, 0xc05: 0x19bd, + 0xc06: 0x22f5, 0xc07: 0x1b94, 0xc08: 0x19c3, 0xc09: 0x194b, 0xc0a: 0x19e1, 0xc0b: 0x194e, + 0xc0c: 0x19ea, 0xc0d: 0x196c, 0xc0e: 0x196f, 0xc0f: 0x1a05, 0xc10: 0x1a0b, 0xc11: 0x1a0e, + 0xc12: 0x1c30, 0xc13: 0x1a11, 0xc14: 0x1a23, 0xc15: 0x1c38, 0xc16: 0x1c44, 0xc17: 0x1990, + 0xc18: 0x1d57, 0xc19: 0x1bbc, 0xc1a: 0x1993, 0xc1b: 0x1a5c, 0xc1c: 0x19a5, 0xc1d: 0x19b4, + 0xc1e: 0x22e2, 0xc1f: 0x22dc, 0xc20: 0x1cc1, 0xc21: 0x1cd0, 0xc22: 0x1cdf, 0xc23: 0x1cee, + 0xc24: 0x1cfd, 0xc25: 0x1d0c, 0xc26: 0x1d1b, 0xc27: 0x1d2a, 0xc28: 0x1d39, 0xc29: 0x2186, + 0xc2a: 0x2198, 0xc2b: 0x21aa, 0xc2c: 0x21bc, 0xc2d: 0x21c8, 0xc2e: 0x21d4, 0xc2f: 0x21e0, + 0xc30: 0x21ec, 0xc31: 0x21f8, 0xc32: 0x2204, 0xc33: 0x2240, 0xc34: 0x224c, 0xc35: 0x2258, + 0xc36: 0x2264, 0xc37: 0x2270, 0xc38: 0x227c, 0xc39: 0x2282, 0xc3a: 0x2288, 0xc3b: 0x228e, + 0xc3c: 0x2294, 0xc3d: 0x22a6, 0xc3e: 0x22ac, 0xc3f: 0x1c10, + // Block 0x31, offset 0xc40 + 0xc40: 0x1377, 0xc41: 0x0cfb, 0xc42: 0x13d3, 0xc43: 0x139f, 0xc44: 0x0e57, 0xc45: 0x06eb, + 0xc46: 0x08df, 0xc47: 0x162b, 0xc48: 0x162b, 0xc49: 0x0a0b, 0xc4a: 0x145f, 0xc4b: 0x0943, + 0xc4c: 0x0a07, 0xc4d: 0x0bef, 0xc4e: 0x0fcf, 0xc4f: 0x115f, 0xc50: 0x1297, 0xc51: 0x12d3, + 0xc52: 0x1307, 0xc53: 0x141b, 0xc54: 0x0d73, 0xc55: 0x0dff, 0xc56: 0x0eab, 0xc57: 0x0f43, + 0xc58: 0x125f, 0xc59: 0x1447, 0xc5a: 0x1573, 0xc5b: 0x070f, 0xc5c: 0x08b3, 0xc5d: 0x0d87, + 0xc5e: 0x0ecf, 0xc5f: 0x1293, 0xc60: 0x15c3, 0xc61: 0x0ab3, 0xc62: 0x0e77, 0xc63: 0x1283, + 0xc64: 0x1317, 0xc65: 0x0c23, 0xc66: 0x11bb, 0xc67: 0x12df, 0xc68: 0x0b1f, 0xc69: 0x0d0f, + 0xc6a: 0x0e17, 0xc6b: 0x0f1b, 0xc6c: 0x1427, 0xc6d: 0x074f, 0xc6e: 0x07e7, 0xc6f: 0x0853, + 0xc70: 0x0c8b, 0xc71: 0x0d7f, 0xc72: 0x0ecb, 0xc73: 0x0fef, 0xc74: 0x1177, 0xc75: 0x128b, + 0xc76: 0x12a3, 0xc77: 0x13c7, 0xc78: 0x14ef, 0xc79: 0x15a3, 0xc7a: 0x15bf, 0xc7b: 0x102b, + 0xc7c: 0x106b, 0xc7d: 0x1123, 0xc7e: 0x1243, 0xc7f: 0x147b, + // Block 0x32, offset 0xc80 + 0xc80: 0x15cb, 0xc81: 0x134b, 0xc82: 0x09c7, 0xc83: 0x0b3b, 0xc84: 0x10db, 0xc85: 0x119b, + 0xc86: 0x0eff, 0xc87: 0x1033, 0xc88: 0x1397, 0xc89: 0x14e7, 0xc8a: 0x09c3, 0xc8b: 0x0a8f, + 0xc8c: 0x0d77, 0xc8d: 0x0e2b, 0xc8e: 0x0e5f, 0xc8f: 0x1113, 0xc90: 0x113b, 0xc91: 0x14a7, + 0xc92: 0x084f, 0xc93: 0x11a7, 0xc94: 0x07f3, 0xc95: 0x07ef, 0xc96: 0x1097, 0xc97: 0x1127, + 0xc98: 0x125b, 0xc99: 0x14af, 0xc9a: 0x1367, 0xc9b: 0x0c27, 0xc9c: 0x0d73, 0xc9d: 0x1357, + 0xc9e: 0x06f7, 0xc9f: 0x0a63, 0xca0: 0x0b93, 0xca1: 0x0f2f, 0xca2: 0x0faf, 0xca3: 0x0873, + 0xca4: 0x103b, 0xca5: 0x075f, 0xca6: 0x0b77, 0xca7: 0x06d7, 0xca8: 0x0deb, 0xca9: 0x0ca3, + 0xcaa: 0x110f, 0xcab: 0x08c7, 0xcac: 0x09b3, 0xcad: 0x0ffb, 0xcae: 0x1263, 0xcaf: 0x133b, + 0xcb0: 0x0db7, 0xcb1: 0x13f7, 0xcb2: 0x0de3, 0xcb3: 0x0c37, 0xcb4: 0x121b, 0xcb5: 0x0c57, + 0xcb6: 0x0fab, 0xcb7: 0x072b, 0xcb8: 0x07a7, 0xcb9: 0x07eb, 0xcba: 0x0d53, 0xcbb: 0x10fb, + 0xcbc: 0x11f3, 0xcbd: 0x1347, 0xcbe: 0x145b, 0xcbf: 0x085b, + // Block 0x33, offset 0xcc0 + 0xcc0: 0x090f, 0xcc1: 0x0a17, 0xcc2: 0x0b2f, 0xcc3: 0x0cbf, 0xcc4: 0x0e7b, 0xcc5: 0x103f, + 0xcc6: 0x1497, 0xcc7: 0x157b, 0xcc8: 0x15cf, 0xcc9: 0x15e7, 0xcca: 0x0837, 0xccb: 0x0cf3, + 0xccc: 0x0da3, 0xccd: 0x13eb, 0xcce: 0x0afb, 0xccf: 0x0bd7, 0xcd0: 0x0bf3, 0xcd1: 0x0c83, + 0xcd2: 0x0e6b, 0xcd3: 0x0eb7, 0xcd4: 0x0f67, 0xcd5: 0x108b, 0xcd6: 0x112f, 0xcd7: 0x1193, + 0xcd8: 0x13db, 0xcd9: 0x126b, 0xcda: 0x1403, 0xcdb: 0x147f, 0xcdc: 0x080f, 0xcdd: 0x083b, + 0xcde: 0x0923, 0xcdf: 0x0ea7, 0xce0: 0x12f3, 0xce1: 0x133b, 0xce2: 0x0b1b, 0xce3: 0x0b8b, + 0xce4: 0x0c4f, 0xce5: 0x0daf, 0xce6: 0x10d7, 0xce7: 0x0f23, 0xce8: 0x073b, 0xce9: 0x097f, + 0xcea: 0x0a63, 0xceb: 0x0ac7, 0xcec: 0x0b97, 0xced: 0x0f3f, 0xcee: 0x0f5b, 0xcef: 0x116b, + 0xcf0: 0x118b, 0xcf1: 0x1463, 0xcf2: 0x14e3, 0xcf3: 0x14f3, 0xcf4: 0x152f, 0xcf5: 0x0753, + 0xcf6: 0x107f, 0xcf7: 0x144f, 0xcf8: 0x14cb, 0xcf9: 0x0baf, 0xcfa: 0x0717, 0xcfb: 0x0777, + 0xcfc: 0x0a67, 0xcfd: 0x0a87, 0xcfe: 0x0caf, 0xcff: 0x0d73, + // Block 0x34, offset 0xd00 + 0xd00: 0x0ec3, 0xd01: 0x0fcb, 0xd02: 0x1277, 0xd03: 0x1417, 0xd04: 0x1623, 0xd05: 0x0ce3, + 0xd06: 0x14a3, 0xd07: 0x0833, 0xd08: 0x0d2f, 0xd09: 0x0d3b, 0xd0a: 0x0e0f, 0xd0b: 0x0e47, + 0xd0c: 0x0f4b, 0xd0d: 0x0fa7, 0xd0e: 0x1027, 0xd0f: 0x110b, 0xd10: 0x153b, 0xd11: 0x07af, + 0xd12: 0x0c03, 0xd13: 0x14b3, 0xd14: 0x0767, 0xd15: 0x0aab, 0xd16: 0x0e2f, 0xd17: 0x13df, + 0xd18: 0x0b67, 0xd19: 0x0bb7, 0xd1a: 0x0d43, 0xd1b: 0x0f2f, 0xd1c: 0x14bb, 0xd1d: 0x0817, + 0xd1e: 0x08ff, 0xd1f: 0x0a97, 0xd20: 0x0cd3, 0xd21: 0x0d1f, 0xd22: 0x0d5f, 0xd23: 0x0df3, + 0xd24: 0x0f47, 0xd25: 0x0fbb, 0xd26: 0x1157, 0xd27: 0x12f7, 0xd28: 0x1303, 0xd29: 0x1457, + 0xd2a: 0x14d7, 0xd2b: 0x0883, 0xd2c: 0x0e4b, 0xd2d: 0x0903, 0xd2e: 0x0ec7, 0xd2f: 0x0f6b, + 0xd30: 0x1287, 0xd31: 0x14bf, 0xd32: 0x15ab, 0xd33: 0x15d3, 0xd34: 0x0d37, 0xd35: 0x0e27, + 0xd36: 0x11c3, 0xd37: 0x10b7, 0xd38: 0x10c3, 0xd39: 0x10e7, 0xd3a: 0x0f17, 0xd3b: 0x0e9f, + 0xd3c: 0x1363, 0xd3d: 0x0733, 0xd3e: 0x122b, 0xd3f: 0x081b, + // Block 0x35, offset 0xd40 + 0xd40: 0x080b, 0xd41: 0x0b0b, 0xd42: 0x0c2b, 0xd43: 0x10f3, 0xd44: 0x0a53, 0xd45: 0x0e03, + 0xd46: 0x0cef, 0xd47: 0x13e7, 0xd48: 0x12e7, 0xd49: 0x14ab, 0xd4a: 0x1323, 0xd4b: 0x0b27, + 0xd4c: 0x0787, 0xd4d: 0x095b, 0xd50: 0x09af, + 0xd52: 0x0cdf, 0xd55: 0x07f7, 0xd56: 0x0f1f, 0xd57: 0x0fe3, + 0xd58: 0x1047, 0xd59: 0x1063, 0xd5a: 0x1067, 0xd5b: 0x107b, 0xd5c: 0x14fb, 0xd5d: 0x10eb, + 0xd5e: 0x116f, 0xd60: 0x128f, 0xd62: 0x1353, + 0xd65: 0x1407, 0xd66: 0x1433, + 0xd6a: 0x154f, 0xd6b: 0x1553, 0xd6c: 0x1557, 0xd6d: 0x15bb, 0xd6e: 0x142b, 0xd6f: 0x14c7, + 0xd70: 0x0757, 0xd71: 0x077b, 0xd72: 0x078f, 0xd73: 0x084b, 0xd74: 0x0857, 0xd75: 0x0897, + 0xd76: 0x094b, 0xd77: 0x0967, 0xd78: 0x096f, 0xd79: 0x09ab, 0xd7a: 0x09b7, 0xd7b: 0x0a93, + 0xd7c: 0x0a9b, 0xd7d: 0x0ba3, 0xd7e: 0x0bcb, 0xd7f: 0x0bd3, + // Block 0x36, offset 0xd80 + 0xd80: 0x0beb, 0xd81: 0x0c97, 0xd82: 0x0cc7, 0xd83: 0x0ce7, 0xd84: 0x0d57, 0xd85: 0x0e1b, + 0xd86: 0x0e37, 0xd87: 0x0e67, 0xd88: 0x0ebb, 0xd89: 0x0edb, 0xd8a: 0x0f4f, 0xd8b: 0x102f, + 0xd8c: 0x104b, 0xd8d: 0x1053, 0xd8e: 0x104f, 0xd8f: 0x1057, 0xd90: 0x105b, 0xd91: 0x105f, + 0xd92: 0x1073, 0xd93: 0x1077, 0xd94: 0x109b, 0xd95: 0x10af, 0xd96: 0x10cb, 0xd97: 0x112f, + 0xd98: 0x1137, 0xd99: 0x113f, 0xd9a: 0x1153, 0xd9b: 0x117b, 0xd9c: 0x11cb, 0xd9d: 0x11ff, + 0xd9e: 0x11ff, 0xd9f: 0x1267, 0xda0: 0x130f, 0xda1: 0x1327, 0xda2: 0x135b, 0xda3: 0x135f, + 0xda4: 0x13a3, 0xda5: 0x13a7, 0xda6: 0x13ff, 0xda7: 0x1407, 0xda8: 0x14db, 0xda9: 0x151f, + 0xdaa: 0x1537, 0xdab: 0x0b9b, 0xdac: 0x171e, 0xdad: 0x11e3, + 0xdb0: 0x06df, 0xdb1: 0x07e3, 0xdb2: 0x07a3, 0xdb3: 0x074b, 0xdb4: 0x078b, 0xdb5: 0x07b7, + 0xdb6: 0x0847, 0xdb7: 0x0863, 0xdb8: 0x094b, 0xdb9: 0x0937, 0xdba: 0x0947, 0xdbb: 0x0963, + 0xdbc: 0x09af, 0xdbd: 0x09bf, 0xdbe: 0x0a03, 0xdbf: 0x0a0f, + // Block 0x37, offset 0xdc0 + 0xdc0: 0x0a2b, 0xdc1: 0x0a3b, 0xdc2: 0x0b23, 0xdc3: 0x0b2b, 0xdc4: 0x0b5b, 0xdc5: 0x0b7b, + 0xdc6: 0x0bab, 0xdc7: 0x0bc3, 0xdc8: 0x0bb3, 0xdc9: 0x0bd3, 0xdca: 0x0bc7, 0xdcb: 0x0beb, + 0xdcc: 0x0c07, 0xdcd: 0x0c5f, 0xdce: 0x0c6b, 0xdcf: 0x0c73, 0xdd0: 0x0c9b, 0xdd1: 0x0cdf, + 0xdd2: 0x0d0f, 0xdd3: 0x0d13, 0xdd4: 0x0d27, 0xdd5: 0x0da7, 0xdd6: 0x0db7, 0xdd7: 0x0e0f, + 0xdd8: 0x0e5b, 0xdd9: 0x0e53, 0xdda: 0x0e67, 0xddb: 0x0e83, 0xddc: 0x0ebb, 0xddd: 0x1013, + 0xdde: 0x0edf, 0xddf: 0x0f13, 0xde0: 0x0f1f, 0xde1: 0x0f5f, 0xde2: 0x0f7b, 0xde3: 0x0f9f, + 0xde4: 0x0fc3, 0xde5: 0x0fc7, 0xde6: 0x0fe3, 0xde7: 0x0fe7, 0xde8: 0x0ff7, 0xde9: 0x100b, + 0xdea: 0x1007, 0xdeb: 0x1037, 0xdec: 0x10b3, 0xded: 0x10cb, 0xdee: 0x10e3, 0xdef: 0x111b, + 0xdf0: 0x112f, 0xdf1: 0x114b, 0xdf2: 0x117b, 0xdf3: 0x122f, 0xdf4: 0x1257, 0xdf5: 0x12cb, + 0xdf6: 0x1313, 0xdf7: 0x131f, 0xdf8: 0x1327, 0xdf9: 0x133f, 0xdfa: 0x1353, 0xdfb: 0x1343, + 0xdfc: 0x135b, 0xdfd: 0x1357, 0xdfe: 0x134f, 0xdff: 0x135f, + // Block 0x38, offset 0xe00 + 0xe00: 0x136b, 0xe01: 0x13a7, 0xe02: 0x13e3, 0xe03: 0x1413, 0xe04: 0x144b, 0xe05: 0x146b, + 0xe06: 0x14b7, 0xe07: 0x14db, 0xe08: 0x14fb, 0xe09: 0x150f, 0xe0a: 0x151f, 0xe0b: 0x152b, + 0xe0c: 0x1537, 0xe0d: 0x158b, 0xe0e: 0x162b, 0xe0f: 0x16b5, 0xe10: 0x16b0, 0xe11: 0x16e2, + 0xe12: 0x0607, 0xe13: 0x062f, 0xe14: 0x0633, 0xe15: 0x1764, 0xe16: 0x1791, 0xe17: 0x1809, + 0xe18: 0x1617, 0xe19: 0x1627, + // Block 0x39, offset 0xe40 + 0xe40: 0x19d5, 0xe41: 0x19d8, 0xe42: 0x19db, 0xe43: 0x1c08, 0xe44: 0x1c0c, 0xe45: 0x1a5f, + 0xe46: 0x1a5f, + 0xe53: 0x1d75, 0xe54: 0x1d66, 0xe55: 0x1d6b, 0xe56: 0x1d7a, 0xe57: 0x1d70, + 0xe5d: 0x4426, + 0xe5e: 0x8115, 0xe5f: 0x4498, 0xe60: 0x022d, 0xe61: 0x0215, 0xe62: 0x021e, 0xe63: 0x0221, + 0xe64: 0x0224, 0xe65: 0x0227, 0xe66: 0x022a, 0xe67: 0x0230, 0xe68: 0x0233, 0xe69: 0x0017, + 0xe6a: 0x4486, 0xe6b: 0x448c, 0xe6c: 0x458a, 0xe6d: 0x4592, 0xe6e: 0x43de, 0xe6f: 0x43e4, + 0xe70: 0x43ea, 0xe71: 0x43f0, 0xe72: 0x43fc, 0xe73: 0x4402, 0xe74: 0x4408, 0xe75: 0x4414, + 0xe76: 0x441a, 0xe78: 0x4420, 0xe79: 0x442c, 0xe7a: 0x4432, 0xe7b: 0x4438, + 0xe7c: 0x4444, 0xe7e: 0x444a, + // Block 0x3a, offset 0xe80 + 0xe80: 0x4450, 0xe81: 0x4456, 0xe83: 0x445c, 0xe84: 0x4462, + 0xe86: 0x446e, 0xe87: 0x4474, 0xe88: 0x447a, 0xe89: 0x4480, 0xe8a: 0x4492, 0xe8b: 0x440e, + 0xe8c: 0x43f6, 0xe8d: 0x443e, 0xe8e: 0x4468, 0xe8f: 0x1d7f, 0xe90: 0x0299, 0xe91: 0x0299, + 0xe92: 0x02a2, 0xe93: 0x02a2, 0xe94: 0x02a2, 0xe95: 0x02a2, 0xe96: 0x02a5, 0xe97: 0x02a5, + 0xe98: 0x02a5, 0xe99: 0x02a5, 0xe9a: 0x02ab, 0xe9b: 0x02ab, 0xe9c: 0x02ab, 0xe9d: 0x02ab, + 0xe9e: 0x029f, 0xe9f: 0x029f, 0xea0: 0x029f, 0xea1: 0x029f, 0xea2: 0x02a8, 0xea3: 0x02a8, + 0xea4: 0x02a8, 0xea5: 0x02a8, 0xea6: 0x029c, 0xea7: 0x029c, 0xea8: 0x029c, 0xea9: 0x029c, + 0xeaa: 0x02cf, 0xeab: 0x02cf, 0xeac: 0x02cf, 0xead: 0x02cf, 0xeae: 0x02d2, 0xeaf: 0x02d2, + 0xeb0: 0x02d2, 0xeb1: 0x02d2, 0xeb2: 0x02b1, 0xeb3: 0x02b1, 0xeb4: 0x02b1, 0xeb5: 0x02b1, + 0xeb6: 0x02ae, 0xeb7: 0x02ae, 0xeb8: 0x02ae, 0xeb9: 0x02ae, 0xeba: 0x02b4, 0xebb: 0x02b4, + 0xebc: 0x02b4, 0xebd: 0x02b4, 0xebe: 0x02b7, 0xebf: 0x02b7, + // Block 0x3b, offset 0xec0 + 0xec0: 0x02b7, 0xec1: 0x02b7, 0xec2: 0x02c0, 0xec3: 0x02c0, 0xec4: 0x02bd, 0xec5: 0x02bd, + 0xec6: 0x02c3, 0xec7: 0x02c3, 0xec8: 0x02ba, 0xec9: 0x02ba, 0xeca: 0x02c9, 0xecb: 0x02c9, + 0xecc: 0x02c6, 0xecd: 0x02c6, 0xece: 0x02d5, 0xecf: 0x02d5, 0xed0: 0x02d5, 0xed1: 0x02d5, + 0xed2: 0x02db, 0xed3: 0x02db, 0xed4: 0x02db, 0xed5: 0x02db, 0xed6: 0x02e1, 0xed7: 0x02e1, + 0xed8: 0x02e1, 0xed9: 0x02e1, 0xeda: 0x02de, 0xedb: 0x02de, 0xedc: 0x02de, 0xedd: 0x02de, + 0xede: 0x02e4, 0xedf: 0x02e4, 0xee0: 0x02e7, 0xee1: 0x02e7, 0xee2: 0x02e7, 0xee3: 0x02e7, + 0xee4: 0x4504, 0xee5: 0x4504, 0xee6: 0x02ed, 0xee7: 0x02ed, 0xee8: 0x02ed, 0xee9: 0x02ed, + 0xeea: 0x02ea, 0xeeb: 0x02ea, 0xeec: 0x02ea, 0xeed: 0x02ea, 0xeee: 0x0308, 0xeef: 0x0308, + 0xef0: 0x44fe, 0xef1: 0x44fe, + // Block 0x3c, offset 0xf00 + 0xf13: 0x02d8, 0xf14: 0x02d8, 0xf15: 0x02d8, 0xf16: 0x02d8, 0xf17: 0x02f6, + 0xf18: 0x02f6, 0xf19: 0x02f3, 0xf1a: 0x02f3, 0xf1b: 0x02f9, 0xf1c: 0x02f9, 0xf1d: 0x204f, + 0xf1e: 0x02ff, 0xf1f: 0x02ff, 0xf20: 0x02f0, 0xf21: 0x02f0, 0xf22: 0x02fc, 0xf23: 0x02fc, + 0xf24: 0x0305, 0xf25: 0x0305, 0xf26: 0x0305, 0xf27: 0x0305, 0xf28: 0x028d, 0xf29: 0x028d, + 0xf2a: 0x25aa, 0xf2b: 0x25aa, 0xf2c: 0x261a, 0xf2d: 0x261a, 0xf2e: 0x25e9, 0xf2f: 0x25e9, + 0xf30: 0x2605, 0xf31: 0x2605, 0xf32: 0x25fe, 0xf33: 0x25fe, 0xf34: 0x260c, 0xf35: 0x260c, + 0xf36: 0x2613, 0xf37: 0x2613, 0xf38: 0x2613, 0xf39: 0x25f0, 0xf3a: 0x25f0, 0xf3b: 0x25f0, + 0xf3c: 0x0302, 0xf3d: 0x0302, 0xf3e: 0x0302, 0xf3f: 0x0302, + // Block 0x3d, offset 0xf40 + 0xf40: 0x25b1, 0xf41: 0x25b8, 0xf42: 0x25d4, 0xf43: 0x25f0, 0xf44: 0x25f7, 0xf45: 0x1d89, + 0xf46: 0x1d8e, 0xf47: 0x1d93, 0xf48: 0x1da2, 0xf49: 0x1db1, 0xf4a: 0x1db6, 0xf4b: 0x1dbb, + 0xf4c: 0x1dc0, 0xf4d: 0x1dc5, 0xf4e: 0x1dd4, 0xf4f: 0x1de3, 0xf50: 0x1de8, 0xf51: 0x1ded, + 0xf52: 0x1dfc, 0xf53: 0x1e0b, 0xf54: 0x1e10, 0xf55: 0x1e15, 0xf56: 0x1e1a, 0xf57: 0x1e29, + 0xf58: 0x1e2e, 0xf59: 0x1e3d, 0xf5a: 0x1e42, 0xf5b: 0x1e47, 0xf5c: 0x1e56, 0xf5d: 0x1e5b, + 0xf5e: 0x1e60, 0xf5f: 0x1e6a, 0xf60: 0x1ea6, 0xf61: 0x1eb5, 0xf62: 0x1ec4, 0xf63: 0x1ec9, + 0xf64: 0x1ece, 0xf65: 0x1ed8, 0xf66: 0x1ee7, 0xf67: 0x1eec, 0xf68: 0x1efb, 0xf69: 0x1f00, + 0xf6a: 0x1f05, 0xf6b: 0x1f14, 0xf6c: 0x1f19, 0xf6d: 0x1f28, 0xf6e: 0x1f2d, 0xf6f: 0x1f32, + 0xf70: 0x1f37, 0xf71: 0x1f3c, 0xf72: 0x1f41, 0xf73: 0x1f46, 0xf74: 0x1f4b, 0xf75: 0x1f50, + 0xf76: 0x1f55, 0xf77: 0x1f5a, 0xf78: 0x1f5f, 0xf79: 0x1f64, 0xf7a: 0x1f69, 0xf7b: 0x1f6e, + 0xf7c: 0x1f73, 0xf7d: 0x1f78, 0xf7e: 0x1f7d, 0xf7f: 0x1f87, + // Block 0x3e, offset 0xf80 + 0xf80: 0x1f8c, 0xf81: 0x1f91, 0xf82: 0x1f96, 0xf83: 0x1fa0, 0xf84: 0x1fa5, 0xf85: 0x1faf, + 0xf86: 0x1fb4, 0xf87: 0x1fb9, 0xf88: 0x1fbe, 0xf89: 0x1fc3, 0xf8a: 0x1fc8, 0xf8b: 0x1fcd, + 0xf8c: 0x1fd2, 0xf8d: 0x1fd7, 0xf8e: 0x1fe6, 0xf8f: 0x1ff5, 0xf90: 0x1ffa, 0xf91: 0x1fff, + 0xf92: 0x2004, 0xf93: 0x2009, 0xf94: 0x200e, 0xf95: 0x2018, 0xf96: 0x201d, 0xf97: 0x2022, + 0xf98: 0x2031, 0xf99: 0x2040, 0xf9a: 0x2045, 0xf9b: 0x44b6, 0xf9c: 0x44bc, 0xf9d: 0x44f2, + 0xf9e: 0x4549, 0xf9f: 0x4550, 0xfa0: 0x4557, 0xfa1: 0x455e, 0xfa2: 0x4565, 0xfa3: 0x456c, + 0xfa4: 0x25c6, 0xfa5: 0x25cd, 0xfa6: 0x25d4, 0xfa7: 0x25db, 0xfa8: 0x25f0, 0xfa9: 0x25f7, + 0xfaa: 0x1d98, 0xfab: 0x1d9d, 0xfac: 0x1da2, 0xfad: 0x1da7, 0xfae: 0x1db1, 0xfaf: 0x1db6, + 0xfb0: 0x1dca, 0xfb1: 0x1dcf, 0xfb2: 0x1dd4, 0xfb3: 0x1dd9, 0xfb4: 0x1de3, 0xfb5: 0x1de8, + 0xfb6: 0x1df2, 0xfb7: 0x1df7, 0xfb8: 0x1dfc, 0xfb9: 0x1e01, 0xfba: 0x1e0b, 0xfbb: 0x1e10, + 0xfbc: 0x1f3c, 0xfbd: 0x1f41, 0xfbe: 0x1f50, 0xfbf: 0x1f55, + // Block 0x3f, offset 0xfc0 + 0xfc0: 0x1f5a, 0xfc1: 0x1f6e, 0xfc2: 0x1f73, 0xfc3: 0x1f78, 0xfc4: 0x1f7d, 0xfc5: 0x1f96, + 0xfc6: 0x1fa0, 0xfc7: 0x1fa5, 0xfc8: 0x1faa, 0xfc9: 0x1fbe, 0xfca: 0x1fdc, 0xfcb: 0x1fe1, + 0xfcc: 0x1fe6, 0xfcd: 0x1feb, 0xfce: 0x1ff5, 0xfcf: 0x1ffa, 0xfd0: 0x44f2, 0xfd1: 0x2027, + 0xfd2: 0x202c, 0xfd3: 0x2031, 0xfd4: 0x2036, 0xfd5: 0x2040, 0xfd6: 0x2045, 0xfd7: 0x25b1, + 0xfd8: 0x25b8, 0xfd9: 0x25bf, 0xfda: 0x25d4, 0xfdb: 0x25e2, 0xfdc: 0x1d89, 0xfdd: 0x1d8e, + 0xfde: 0x1d93, 0xfdf: 0x1da2, 0xfe0: 0x1dac, 0xfe1: 0x1dbb, 0xfe2: 0x1dc0, 0xfe3: 0x1dc5, + 0xfe4: 0x1dd4, 0xfe5: 0x1dde, 0xfe6: 0x1dfc, 0xfe7: 0x1e15, 0xfe8: 0x1e1a, 0xfe9: 0x1e29, + 0xfea: 0x1e2e, 0xfeb: 0x1e3d, 0xfec: 0x1e47, 0xfed: 0x1e56, 0xfee: 0x1e5b, 0xfef: 0x1e60, + 0xff0: 0x1e6a, 0xff1: 0x1ea6, 0xff2: 0x1eab, 0xff3: 0x1eb5, 0xff4: 0x1ec4, 0xff5: 0x1ec9, + 0xff6: 0x1ece, 0xff7: 0x1ed8, 0xff8: 0x1ee7, 0xff9: 0x1efb, 0xffa: 0x1f00, 0xffb: 0x1f05, + 0xffc: 0x1f14, 0xffd: 0x1f19, 0xffe: 0x1f28, 0xfff: 0x1f2d, + // Block 0x40, offset 0x1000 + 0x1000: 0x1f32, 0x1001: 0x1f37, 0x1002: 0x1f46, 0x1003: 0x1f4b, 0x1004: 0x1f5f, 0x1005: 0x1f64, + 0x1006: 0x1f69, 0x1007: 0x1f6e, 0x1008: 0x1f73, 0x1009: 0x1f87, 0x100a: 0x1f8c, 0x100b: 0x1f91, + 0x100c: 0x1f96, 0x100d: 0x1f9b, 0x100e: 0x1faf, 0x100f: 0x1fb4, 0x1010: 0x1fb9, 0x1011: 0x1fbe, + 0x1012: 0x1fcd, 0x1013: 0x1fd2, 0x1014: 0x1fd7, 0x1015: 0x1fe6, 0x1016: 0x1ff0, 0x1017: 0x1fff, + 0x1018: 0x2004, 0x1019: 0x44e6, 0x101a: 0x2018, 0x101b: 0x201d, 0x101c: 0x2022, 0x101d: 0x2031, + 0x101e: 0x203b, 0x101f: 0x25d4, 0x1020: 0x25e2, 0x1021: 0x1da2, 0x1022: 0x1dac, 0x1023: 0x1dd4, + 0x1024: 0x1dde, 0x1025: 0x1dfc, 0x1026: 0x1e06, 0x1027: 0x1e6a, 0x1028: 0x1e6f, 0x1029: 0x1e92, + 0x102a: 0x1e97, 0x102b: 0x1f6e, 0x102c: 0x1f73, 0x102d: 0x1f96, 0x102e: 0x1fe6, 0x102f: 0x1ff0, + 0x1030: 0x2031, 0x1031: 0x203b, 0x1032: 0x459a, 0x1033: 0x45a2, 0x1034: 0x45aa, 0x1035: 0x1ef1, + 0x1036: 0x1ef6, 0x1037: 0x1f0a, 0x1038: 0x1f0f, 0x1039: 0x1f1e, 0x103a: 0x1f23, 0x103b: 0x1e74, + 0x103c: 0x1e79, 0x103d: 0x1e9c, 0x103e: 0x1ea1, 0x103f: 0x1e33, + // Block 0x41, offset 0x1040 + 0x1040: 0x1e38, 0x1041: 0x1e1f, 0x1042: 0x1e24, 0x1043: 0x1e4c, 0x1044: 0x1e51, 0x1045: 0x1eba, + 0x1046: 0x1ebf, 0x1047: 0x1edd, 0x1048: 0x1ee2, 0x1049: 0x1e7e, 0x104a: 0x1e83, 0x104b: 0x1e88, + 0x104c: 0x1e92, 0x104d: 0x1e8d, 0x104e: 0x1e65, 0x104f: 0x1eb0, 0x1050: 0x1ed3, 0x1051: 0x1ef1, + 0x1052: 0x1ef6, 0x1053: 0x1f0a, 0x1054: 0x1f0f, 0x1055: 0x1f1e, 0x1056: 0x1f23, 0x1057: 0x1e74, + 0x1058: 0x1e79, 0x1059: 0x1e9c, 0x105a: 0x1ea1, 0x105b: 0x1e33, 0x105c: 0x1e38, 0x105d: 0x1e1f, + 0x105e: 0x1e24, 0x105f: 0x1e4c, 0x1060: 0x1e51, 0x1061: 0x1eba, 0x1062: 0x1ebf, 0x1063: 0x1edd, + 0x1064: 0x1ee2, 0x1065: 0x1e7e, 0x1066: 0x1e83, 0x1067: 0x1e88, 0x1068: 0x1e92, 0x1069: 0x1e8d, + 0x106a: 0x1e65, 0x106b: 0x1eb0, 0x106c: 0x1ed3, 0x106d: 0x1e7e, 0x106e: 0x1e83, 0x106f: 0x1e88, + 0x1070: 0x1e92, 0x1071: 0x1e6f, 0x1072: 0x1e97, 0x1073: 0x1eec, 0x1074: 0x1e56, 0x1075: 0x1e5b, + 0x1076: 0x1e60, 0x1077: 0x1e7e, 0x1078: 0x1e83, 0x1079: 0x1e88, 0x107a: 0x1eec, 0x107b: 0x1efb, + 0x107c: 0x449e, 0x107d: 0x449e, + // Block 0x42, offset 0x1080 + 0x1090: 0x2311, 0x1091: 0x2326, + 0x1092: 0x2326, 0x1093: 0x232d, 0x1094: 0x2334, 0x1095: 0x2349, 0x1096: 0x2350, 0x1097: 0x2357, + 0x1098: 0x237a, 0x1099: 0x237a, 0x109a: 0x239d, 0x109b: 0x2396, 0x109c: 0x23b2, 0x109d: 0x23a4, + 0x109e: 0x23ab, 0x109f: 0x23ce, 0x10a0: 0x23ce, 0x10a1: 0x23c7, 0x10a2: 0x23d5, 0x10a3: 0x23d5, + 0x10a4: 0x23ff, 0x10a5: 0x23ff, 0x10a6: 0x241b, 0x10a7: 0x23e3, 0x10a8: 0x23e3, 0x10a9: 0x23dc, + 0x10aa: 0x23f1, 0x10ab: 0x23f1, 0x10ac: 0x23f8, 0x10ad: 0x23f8, 0x10ae: 0x2422, 0x10af: 0x2430, + 0x10b0: 0x2430, 0x10b1: 0x2437, 0x10b2: 0x2437, 0x10b3: 0x243e, 0x10b4: 0x2445, 0x10b5: 0x244c, + 0x10b6: 0x2453, 0x10b7: 0x2453, 0x10b8: 0x245a, 0x10b9: 0x2468, 0x10ba: 0x2476, 0x10bb: 0x246f, + 0x10bc: 0x247d, 0x10bd: 0x247d, 0x10be: 0x2492, 0x10bf: 0x2499, + // Block 0x43, offset 0x10c0 + 0x10c0: 0x24ca, 0x10c1: 0x24d8, 0x10c2: 0x24d1, 0x10c3: 0x24b5, 0x10c4: 0x24b5, 0x10c5: 0x24df, + 0x10c6: 0x24df, 0x10c7: 0x24e6, 0x10c8: 0x24e6, 0x10c9: 0x2510, 0x10ca: 0x2517, 0x10cb: 0x251e, + 0x10cc: 0x24f4, 0x10cd: 0x2502, 0x10ce: 0x2525, 0x10cf: 0x252c, + 0x10d2: 0x24fb, 0x10d3: 0x2580, 0x10d4: 0x2587, 0x10d5: 0x255d, 0x10d6: 0x2564, 0x10d7: 0x2548, + 0x10d8: 0x2548, 0x10d9: 0x254f, 0x10da: 0x2579, 0x10db: 0x2572, 0x10dc: 0x259c, 0x10dd: 0x259c, + 0x10de: 0x230a, 0x10df: 0x231f, 0x10e0: 0x2318, 0x10e1: 0x2342, 0x10e2: 0x233b, 0x10e3: 0x2365, + 0x10e4: 0x235e, 0x10e5: 0x2388, 0x10e6: 0x236c, 0x10e7: 0x2381, 0x10e8: 0x23b9, 0x10e9: 0x2406, + 0x10ea: 0x23ea, 0x10eb: 0x2429, 0x10ec: 0x24c3, 0x10ed: 0x24ed, 0x10ee: 0x2595, 0x10ef: 0x258e, + 0x10f0: 0x25a3, 0x10f1: 0x253a, 0x10f2: 0x24a0, 0x10f3: 0x256b, 0x10f4: 0x2492, 0x10f5: 0x24ca, + 0x10f6: 0x2461, 0x10f7: 0x24ae, 0x10f8: 0x2541, 0x10f9: 0x2533, 0x10fa: 0x24bc, 0x10fb: 0x24a7, + 0x10fc: 0x24bc, 0x10fd: 0x2541, 0x10fe: 0x2373, 0x10ff: 0x238f, + // Block 0x44, offset 0x1100 + 0x1100: 0x2509, 0x1101: 0x2484, 0x1102: 0x2303, 0x1103: 0x24a7, 0x1104: 0x244c, 0x1105: 0x241b, + 0x1106: 0x23c0, 0x1107: 0x2556, + 0x1130: 0x2414, 0x1131: 0x248b, 0x1132: 0x27bf, 0x1133: 0x27b6, 0x1134: 0x27ec, 0x1135: 0x27da, + 0x1136: 0x27c8, 0x1137: 0x27e3, 0x1138: 0x27f5, 0x1139: 0x240d, 0x113a: 0x2c7c, 0x113b: 0x2afc, + 0x113c: 0x27d1, + // Block 0x45, offset 0x1140 + 0x1150: 0x0019, 0x1151: 0x0483, + 0x1152: 0x0487, 0x1153: 0x0035, 0x1154: 0x0037, 0x1155: 0x0003, 0x1156: 0x003f, 0x1157: 0x04bf, + 0x1158: 0x04c3, 0x1159: 0x1b5c, + 0x1160: 0x8132, 0x1161: 0x8132, 0x1162: 0x8132, 0x1163: 0x8132, + 0x1164: 0x8132, 0x1165: 0x8132, 0x1166: 0x8132, 0x1167: 0x812d, 0x1168: 0x812d, 0x1169: 0x812d, + 0x116a: 0x812d, 0x116b: 0x812d, 0x116c: 0x812d, 0x116d: 0x812d, 0x116e: 0x8132, 0x116f: 0x8132, + 0x1170: 0x1873, 0x1171: 0x0443, 0x1172: 0x043f, 0x1173: 0x007f, 0x1174: 0x007f, 0x1175: 0x0011, + 0x1176: 0x0013, 0x1177: 0x00b7, 0x1178: 0x00bb, 0x1179: 0x04b7, 0x117a: 0x04bb, 0x117b: 0x04ab, + 0x117c: 0x04af, 0x117d: 0x0493, 0x117e: 0x0497, 0x117f: 0x048b, + // Block 0x46, offset 0x1180 + 0x1180: 0x048f, 0x1181: 0x049b, 0x1182: 0x049f, 0x1183: 0x04a3, 0x1184: 0x04a7, + 0x1187: 0x0077, 0x1188: 0x007b, 0x1189: 0x4269, 0x118a: 0x4269, 0x118b: 0x4269, + 0x118c: 0x4269, 0x118d: 0x007f, 0x118e: 0x007f, 0x118f: 0x007f, 0x1190: 0x0019, 0x1191: 0x0483, + 0x1192: 0x001d, 0x1194: 0x0037, 0x1195: 0x0035, 0x1196: 0x003f, 0x1197: 0x0003, + 0x1198: 0x0443, 0x1199: 0x0011, 0x119a: 0x0013, 0x119b: 0x00b7, 0x119c: 0x00bb, 0x119d: 0x04b7, + 0x119e: 0x04bb, 0x119f: 0x0007, 0x11a0: 0x000d, 0x11a1: 0x0015, 0x11a2: 0x0017, 0x11a3: 0x001b, + 0x11a4: 0x0039, 0x11a5: 0x003d, 0x11a6: 0x003b, 0x11a8: 0x0079, 0x11a9: 0x0009, + 0x11aa: 0x000b, 0x11ab: 0x0041, + 0x11b0: 0x42aa, 0x11b1: 0x44c2, 0x11b2: 0x42af, 0x11b4: 0x42b4, + 0x11b6: 0x42b9, 0x11b7: 0x44c8, 0x11b8: 0x42be, 0x11b9: 0x44ce, 0x11ba: 0x42c3, 0x11bb: 0x44d4, + 0x11bc: 0x42c8, 0x11bd: 0x44da, 0x11be: 0x42cd, 0x11bf: 0x44e0, + // Block 0x47, offset 0x11c0 + 0x11c0: 0x0236, 0x11c1: 0x44a4, 0x11c2: 0x44a4, 0x11c3: 0x44aa, 0x11c4: 0x44aa, 0x11c5: 0x44ec, + 0x11c6: 0x44ec, 0x11c7: 0x44b0, 0x11c8: 0x44b0, 0x11c9: 0x44f8, 0x11ca: 0x44f8, 0x11cb: 0x44f8, + 0x11cc: 0x44f8, 0x11cd: 0x0239, 0x11ce: 0x0239, 0x11cf: 0x023c, 0x11d0: 0x023c, 0x11d1: 0x023c, + 0x11d2: 0x023c, 0x11d3: 0x023f, 0x11d4: 0x023f, 0x11d5: 0x0242, 0x11d6: 0x0242, 0x11d7: 0x0242, + 0x11d8: 0x0242, 0x11d9: 0x0245, 0x11da: 0x0245, 0x11db: 0x0245, 0x11dc: 0x0245, 0x11dd: 0x0248, + 0x11de: 0x0248, 0x11df: 0x0248, 0x11e0: 0x0248, 0x11e1: 0x024b, 0x11e2: 0x024b, 0x11e3: 0x024b, + 0x11e4: 0x024b, 0x11e5: 0x024e, 0x11e6: 0x024e, 0x11e7: 0x024e, 0x11e8: 0x024e, 0x11e9: 0x0251, + 0x11ea: 0x0251, 0x11eb: 0x0254, 0x11ec: 0x0254, 0x11ed: 0x0257, 0x11ee: 0x0257, 0x11ef: 0x025a, + 0x11f0: 0x025a, 0x11f1: 0x025d, 0x11f2: 0x025d, 0x11f3: 0x025d, 0x11f4: 0x025d, 0x11f5: 0x0260, + 0x11f6: 0x0260, 0x11f7: 0x0260, 0x11f8: 0x0260, 0x11f9: 0x0263, 0x11fa: 0x0263, 0x11fb: 0x0263, + 0x11fc: 0x0263, 0x11fd: 0x0266, 0x11fe: 0x0266, 0x11ff: 0x0266, + // Block 0x48, offset 0x1200 + 0x1200: 0x0266, 0x1201: 0x0269, 0x1202: 0x0269, 0x1203: 0x0269, 0x1204: 0x0269, 0x1205: 0x026c, + 0x1206: 0x026c, 0x1207: 0x026c, 0x1208: 0x026c, 0x1209: 0x026f, 0x120a: 0x026f, 0x120b: 0x026f, + 0x120c: 0x026f, 0x120d: 0x0272, 0x120e: 0x0272, 0x120f: 0x0272, 0x1210: 0x0272, 0x1211: 0x0275, + 0x1212: 0x0275, 0x1213: 0x0275, 0x1214: 0x0275, 0x1215: 0x0278, 0x1216: 0x0278, 0x1217: 0x0278, + 0x1218: 0x0278, 0x1219: 0x027b, 0x121a: 0x027b, 0x121b: 0x027b, 0x121c: 0x027b, 0x121d: 0x027e, + 0x121e: 0x027e, 0x121f: 0x027e, 0x1220: 0x027e, 0x1221: 0x0281, 0x1222: 0x0281, 0x1223: 0x0281, + 0x1224: 0x0281, 0x1225: 0x0284, 0x1226: 0x0284, 0x1227: 0x0284, 0x1228: 0x0284, 0x1229: 0x0287, + 0x122a: 0x0287, 0x122b: 0x0287, 0x122c: 0x0287, 0x122d: 0x028a, 0x122e: 0x028a, 0x122f: 0x028d, + 0x1230: 0x028d, 0x1231: 0x0290, 0x1232: 0x0290, 0x1233: 0x0290, 0x1234: 0x0290, 0x1235: 0x2e00, + 0x1236: 0x2e00, 0x1237: 0x2e08, 0x1238: 0x2e08, 0x1239: 0x2e10, 0x123a: 0x2e10, 0x123b: 0x1f82, + 0x123c: 0x1f82, + // Block 0x49, offset 0x1240 + 0x1240: 0x0081, 0x1241: 0x0083, 0x1242: 0x0085, 0x1243: 0x0087, 0x1244: 0x0089, 0x1245: 0x008b, + 0x1246: 0x008d, 0x1247: 0x008f, 0x1248: 0x0091, 0x1249: 0x0093, 0x124a: 0x0095, 0x124b: 0x0097, + 0x124c: 0x0099, 0x124d: 0x009b, 0x124e: 0x009d, 0x124f: 0x009f, 0x1250: 0x00a1, 0x1251: 0x00a3, + 0x1252: 0x00a5, 0x1253: 0x00a7, 0x1254: 0x00a9, 0x1255: 0x00ab, 0x1256: 0x00ad, 0x1257: 0x00af, + 0x1258: 0x00b1, 0x1259: 0x00b3, 0x125a: 0x00b5, 0x125b: 0x00b7, 0x125c: 0x00b9, 0x125d: 0x00bb, + 0x125e: 0x00bd, 0x125f: 0x0477, 0x1260: 0x047b, 0x1261: 0x0487, 0x1262: 0x049b, 0x1263: 0x049f, + 0x1264: 0x0483, 0x1265: 0x05ab, 0x1266: 0x05a3, 0x1267: 0x04c7, 0x1268: 0x04cf, 0x1269: 0x04d7, + 0x126a: 0x04df, 0x126b: 0x04e7, 0x126c: 0x056b, 0x126d: 0x0573, 0x126e: 0x057b, 0x126f: 0x051f, + 0x1270: 0x05af, 0x1271: 0x04cb, 0x1272: 0x04d3, 0x1273: 0x04db, 0x1274: 0x04e3, 0x1275: 0x04eb, + 0x1276: 0x04ef, 0x1277: 0x04f3, 0x1278: 0x04f7, 0x1279: 0x04fb, 0x127a: 0x04ff, 0x127b: 0x0503, + 0x127c: 0x0507, 0x127d: 0x050b, 0x127e: 0x050f, 0x127f: 0x0513, + // Block 0x4a, offset 0x1280 + 0x1280: 0x0517, 0x1281: 0x051b, 0x1282: 0x0523, 0x1283: 0x0527, 0x1284: 0x052b, 0x1285: 0x052f, + 0x1286: 0x0533, 0x1287: 0x0537, 0x1288: 0x053b, 0x1289: 0x053f, 0x128a: 0x0543, 0x128b: 0x0547, + 0x128c: 0x054b, 0x128d: 0x054f, 0x128e: 0x0553, 0x128f: 0x0557, 0x1290: 0x055b, 0x1291: 0x055f, + 0x1292: 0x0563, 0x1293: 0x0567, 0x1294: 0x056f, 0x1295: 0x0577, 0x1296: 0x057f, 0x1297: 0x0583, + 0x1298: 0x0587, 0x1299: 0x058b, 0x129a: 0x058f, 0x129b: 0x0593, 0x129c: 0x0597, 0x129d: 0x05a7, + 0x129e: 0x4a5a, 0x129f: 0x4a60, 0x12a0: 0x03c3, 0x12a1: 0x0313, 0x12a2: 0x0317, 0x12a3: 0x4345, + 0x12a4: 0x031b, 0x12a5: 0x434a, 0x12a6: 0x434f, 0x12a7: 0x031f, 0x12a8: 0x0323, 0x12a9: 0x0327, + 0x12aa: 0x4354, 0x12ab: 0x4359, 0x12ac: 0x435e, 0x12ad: 0x4363, 0x12ae: 0x4368, 0x12af: 0x436d, + 0x12b0: 0x0367, 0x12b1: 0x032b, 0x12b2: 0x032f, 0x12b3: 0x0333, 0x12b4: 0x037b, 0x12b5: 0x0337, + 0x12b6: 0x033b, 0x12b7: 0x033f, 0x12b8: 0x0343, 0x12b9: 0x0347, 0x12ba: 0x034b, 0x12bb: 0x034f, + 0x12bc: 0x0353, 0x12bd: 0x0357, 0x12be: 0x035b, + // Block 0x4b, offset 0x12c0 + 0x12c2: 0x42dc, 0x12c3: 0x42e1, 0x12c4: 0x42e6, 0x12c5: 0x42eb, + 0x12c6: 0x42f0, 0x12c7: 0x42f5, 0x12ca: 0x42fa, 0x12cb: 0x42ff, + 0x12cc: 0x4304, 0x12cd: 0x4309, 0x12ce: 0x430e, 0x12cf: 0x4313, + 0x12d2: 0x4318, 0x12d3: 0x431d, 0x12d4: 0x4322, 0x12d5: 0x4327, 0x12d6: 0x432c, 0x12d7: 0x4331, + 0x12da: 0x4336, 0x12db: 0x433b, 0x12dc: 0x4340, + 0x12e0: 0x00bf, 0x12e1: 0x00c2, 0x12e2: 0x00cb, 0x12e3: 0x4264, + 0x12e4: 0x00c8, 0x12e5: 0x00c5, 0x12e6: 0x0447, 0x12e8: 0x046b, 0x12e9: 0x044b, + 0x12ea: 0x044f, 0x12eb: 0x0453, 0x12ec: 0x0457, 0x12ed: 0x046f, 0x12ee: 0x0473, + // Block 0x4c, offset 0x1300 + 0x1300: 0x0063, 0x1301: 0x0065, 0x1302: 0x0067, 0x1303: 0x0069, 0x1304: 0x006b, 0x1305: 0x006d, + 0x1306: 0x006f, 0x1307: 0x0071, 0x1308: 0x0073, 0x1309: 0x0075, 0x130a: 0x0083, 0x130b: 0x0085, + 0x130c: 0x0087, 0x130d: 0x0089, 0x130e: 0x008b, 0x130f: 0x008d, 0x1310: 0x008f, 0x1311: 0x0091, + 0x1312: 0x0093, 0x1313: 0x0095, 0x1314: 0x0097, 0x1315: 0x0099, 0x1316: 0x009b, 0x1317: 0x009d, + 0x1318: 0x009f, 0x1319: 0x00a1, 0x131a: 0x00a3, 0x131b: 0x00a5, 0x131c: 0x00a7, 0x131d: 0x00a9, + 0x131e: 0x00ab, 0x131f: 0x00ad, 0x1320: 0x00af, 0x1321: 0x00b1, 0x1322: 0x00b3, 0x1323: 0x00b5, + 0x1324: 0x00dd, 0x1325: 0x00f2, 0x1328: 0x0173, 0x1329: 0x0176, + 0x132a: 0x0179, 0x132b: 0x017c, 0x132c: 0x017f, 0x132d: 0x0182, 0x132e: 0x0185, 0x132f: 0x0188, + 0x1330: 0x018b, 0x1331: 0x018e, 0x1332: 0x0191, 0x1333: 0x0194, 0x1334: 0x0197, 0x1335: 0x019a, + 0x1336: 0x019d, 0x1337: 0x01a0, 0x1338: 0x01a3, 0x1339: 0x0188, 0x133a: 0x01a6, 0x133b: 0x01a9, + 0x133c: 0x01ac, 0x133d: 0x01af, 0x133e: 0x01b2, 0x133f: 0x01b5, + // Block 0x4d, offset 0x1340 + 0x1340: 0x01fd, 0x1341: 0x0200, 0x1342: 0x0203, 0x1343: 0x045b, 0x1344: 0x01c7, 0x1345: 0x01d0, + 0x1346: 0x01d6, 0x1347: 0x01fa, 0x1348: 0x01eb, 0x1349: 0x01e8, 0x134a: 0x0206, 0x134b: 0x0209, + 0x134e: 0x0021, 0x134f: 0x0023, 0x1350: 0x0025, 0x1351: 0x0027, + 0x1352: 0x0029, 0x1353: 0x002b, 0x1354: 0x002d, 0x1355: 0x002f, 0x1356: 0x0031, 0x1357: 0x0033, + 0x1358: 0x0021, 0x1359: 0x0023, 0x135a: 0x0025, 0x135b: 0x0027, 0x135c: 0x0029, 0x135d: 0x002b, + 0x135e: 0x002d, 0x135f: 0x002f, 0x1360: 0x0031, 0x1361: 0x0033, 0x1362: 0x0021, 0x1363: 0x0023, + 0x1364: 0x0025, 0x1365: 0x0027, 0x1366: 0x0029, 0x1367: 0x002b, 0x1368: 0x002d, 0x1369: 0x002f, + 0x136a: 0x0031, 0x136b: 0x0033, 0x136c: 0x0021, 0x136d: 0x0023, 0x136e: 0x0025, 0x136f: 0x0027, + 0x1370: 0x0029, 0x1371: 0x002b, 0x1372: 0x002d, 0x1373: 0x002f, 0x1374: 0x0031, 0x1375: 0x0033, + 0x1376: 0x0021, 0x1377: 0x0023, 0x1378: 0x0025, 0x1379: 0x0027, 0x137a: 0x0029, 0x137b: 0x002b, + 0x137c: 0x002d, 0x137d: 0x002f, 0x137e: 0x0031, 0x137f: 0x0033, + // Block 0x4e, offset 0x1380 + 0x1380: 0x0239, 0x1381: 0x023c, 0x1382: 0x0248, 0x1383: 0x0251, 0x1385: 0x028a, + 0x1386: 0x025a, 0x1387: 0x024b, 0x1388: 0x0269, 0x1389: 0x0290, 0x138a: 0x027b, 0x138b: 0x027e, + 0x138c: 0x0281, 0x138d: 0x0284, 0x138e: 0x025d, 0x138f: 0x026f, 0x1390: 0x0275, 0x1391: 0x0263, + 0x1392: 0x0278, 0x1393: 0x0257, 0x1394: 0x0260, 0x1395: 0x0242, 0x1396: 0x0245, 0x1397: 0x024e, + 0x1398: 0x0254, 0x1399: 0x0266, 0x139a: 0x026c, 0x139b: 0x0272, 0x139c: 0x0293, 0x139d: 0x02e4, + 0x139e: 0x02cc, 0x139f: 0x0296, 0x13a1: 0x023c, 0x13a2: 0x0248, + 0x13a4: 0x0287, 0x13a7: 0x024b, 0x13a9: 0x0290, + 0x13aa: 0x027b, 0x13ab: 0x027e, 0x13ac: 0x0281, 0x13ad: 0x0284, 0x13ae: 0x025d, 0x13af: 0x026f, + 0x13b0: 0x0275, 0x13b1: 0x0263, 0x13b2: 0x0278, 0x13b4: 0x0260, 0x13b5: 0x0242, + 0x13b6: 0x0245, 0x13b7: 0x024e, 0x13b9: 0x0266, 0x13bb: 0x0272, + // Block 0x4f, offset 0x13c0 + 0x13c2: 0x0248, + 0x13c7: 0x024b, 0x13c9: 0x0290, 0x13cb: 0x027e, + 0x13cd: 0x0284, 0x13ce: 0x025d, 0x13cf: 0x026f, 0x13d1: 0x0263, + 0x13d2: 0x0278, 0x13d4: 0x0260, 0x13d7: 0x024e, + 0x13d9: 0x0266, 0x13db: 0x0272, 0x13dd: 0x02e4, + 0x13df: 0x0296, 0x13e1: 0x023c, 0x13e2: 0x0248, + 0x13e4: 0x0287, 0x13e7: 0x024b, 0x13e8: 0x0269, 0x13e9: 0x0290, + 0x13ea: 0x027b, 0x13ec: 0x0281, 0x13ed: 0x0284, 0x13ee: 0x025d, 0x13ef: 0x026f, + 0x13f0: 0x0275, 0x13f1: 0x0263, 0x13f2: 0x0278, 0x13f4: 0x0260, 0x13f5: 0x0242, + 0x13f6: 0x0245, 0x13f7: 0x024e, 0x13f9: 0x0266, 0x13fa: 0x026c, 0x13fb: 0x0272, + 0x13fc: 0x0293, 0x13fe: 0x02cc, + // Block 0x50, offset 0x1400 + 0x1400: 0x0239, 0x1401: 0x023c, 0x1402: 0x0248, 0x1403: 0x0251, 0x1404: 0x0287, 0x1405: 0x028a, + 0x1406: 0x025a, 0x1407: 0x024b, 0x1408: 0x0269, 0x1409: 0x0290, 0x140b: 0x027e, + 0x140c: 0x0281, 0x140d: 0x0284, 0x140e: 0x025d, 0x140f: 0x026f, 0x1410: 0x0275, 0x1411: 0x0263, + 0x1412: 0x0278, 0x1413: 0x0257, 0x1414: 0x0260, 0x1415: 0x0242, 0x1416: 0x0245, 0x1417: 0x024e, + 0x1418: 0x0254, 0x1419: 0x0266, 0x141a: 0x026c, 0x141b: 0x0272, + 0x1421: 0x023c, 0x1422: 0x0248, 0x1423: 0x0251, + 0x1425: 0x028a, 0x1426: 0x025a, 0x1427: 0x024b, 0x1428: 0x0269, 0x1429: 0x0290, + 0x142b: 0x027e, 0x142c: 0x0281, 0x142d: 0x0284, 0x142e: 0x025d, 0x142f: 0x026f, + 0x1430: 0x0275, 0x1431: 0x0263, 0x1432: 0x0278, 0x1433: 0x0257, 0x1434: 0x0260, 0x1435: 0x0242, + 0x1436: 0x0245, 0x1437: 0x024e, 0x1438: 0x0254, 0x1439: 0x0266, 0x143a: 0x026c, 0x143b: 0x0272, + // Block 0x51, offset 0x1440 + 0x1440: 0x1879, 0x1441: 0x1876, 0x1442: 0x187c, 0x1443: 0x18a0, 0x1444: 0x18c4, 0x1445: 0x18e8, + 0x1446: 0x190c, 0x1447: 0x1915, 0x1448: 0x191b, 0x1449: 0x1921, 0x144a: 0x1927, + 0x1450: 0x1a8c, 0x1451: 0x1a90, + 0x1452: 0x1a94, 0x1453: 0x1a98, 0x1454: 0x1a9c, 0x1455: 0x1aa0, 0x1456: 0x1aa4, 0x1457: 0x1aa8, + 0x1458: 0x1aac, 0x1459: 0x1ab0, 0x145a: 0x1ab4, 0x145b: 0x1ab8, 0x145c: 0x1abc, 0x145d: 0x1ac0, + 0x145e: 0x1ac4, 0x145f: 0x1ac8, 0x1460: 0x1acc, 0x1461: 0x1ad0, 0x1462: 0x1ad4, 0x1463: 0x1ad8, + 0x1464: 0x1adc, 0x1465: 0x1ae0, 0x1466: 0x1ae4, 0x1467: 0x1ae8, 0x1468: 0x1aec, 0x1469: 0x1af0, + 0x146a: 0x271e, 0x146b: 0x0047, 0x146c: 0x0065, 0x146d: 0x193c, 0x146e: 0x19b1, + 0x1470: 0x0043, 0x1471: 0x0045, 0x1472: 0x0047, 0x1473: 0x0049, 0x1474: 0x004b, 0x1475: 0x004d, + 0x1476: 0x004f, 0x1477: 0x0051, 0x1478: 0x0053, 0x1479: 0x0055, 0x147a: 0x0057, 0x147b: 0x0059, + 0x147c: 0x005b, 0x147d: 0x005d, 0x147e: 0x005f, 0x147f: 0x0061, + // Block 0x52, offset 0x1480 + 0x1480: 0x26ad, 0x1481: 0x26c2, 0x1482: 0x0503, + 0x1490: 0x0c0f, 0x1491: 0x0a47, + 0x1492: 0x08d3, 0x1493: 0x465a, 0x1494: 0x071b, 0x1495: 0x09ef, 0x1496: 0x132f, 0x1497: 0x09ff, + 0x1498: 0x0727, 0x1499: 0x0cd7, 0x149a: 0x0eaf, 0x149b: 0x0caf, 0x149c: 0x0827, 0x149d: 0x0b6b, + 0x149e: 0x07bf, 0x149f: 0x0cb7, 0x14a0: 0x0813, 0x14a1: 0x1117, 0x14a2: 0x0f83, 0x14a3: 0x138b, + 0x14a4: 0x09d3, 0x14a5: 0x090b, 0x14a6: 0x0e63, 0x14a7: 0x0c1b, 0x14a8: 0x0c47, 0x14a9: 0x06bf, + 0x14aa: 0x06cb, 0x14ab: 0x140b, 0x14ac: 0x0adb, 0x14ad: 0x06e7, 0x14ae: 0x08ef, 0x14af: 0x0c3b, + 0x14b0: 0x13b3, 0x14b1: 0x0c13, 0x14b2: 0x106f, 0x14b3: 0x10ab, 0x14b4: 0x08f7, 0x14b5: 0x0e43, + 0x14b6: 0x0d0b, 0x14b7: 0x0d07, 0x14b8: 0x0f97, 0x14b9: 0x082b, 0x14ba: 0x0957, 0x14bb: 0x1443, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x06fb, 0x14c1: 0x06f3, 0x14c2: 0x0703, 0x14c3: 0x1647, 0x14c4: 0x0747, 0x14c5: 0x0757, + 0x14c6: 0x075b, 0x14c7: 0x0763, 0x14c8: 0x076b, 0x14c9: 0x076f, 0x14ca: 0x077b, 0x14cb: 0x0773, + 0x14cc: 0x05b3, 0x14cd: 0x165b, 0x14ce: 0x078f, 0x14cf: 0x0793, 0x14d0: 0x0797, 0x14d1: 0x07b3, + 0x14d2: 0x164c, 0x14d3: 0x05b7, 0x14d4: 0x079f, 0x14d5: 0x07bf, 0x14d6: 0x1656, 0x14d7: 0x07cf, + 0x14d8: 0x07d7, 0x14d9: 0x0737, 0x14da: 0x07df, 0x14db: 0x07e3, 0x14dc: 0x1831, 0x14dd: 0x07ff, + 0x14de: 0x0807, 0x14df: 0x05bf, 0x14e0: 0x081f, 0x14e1: 0x0823, 0x14e2: 0x082b, 0x14e3: 0x082f, + 0x14e4: 0x05c3, 0x14e5: 0x0847, 0x14e6: 0x084b, 0x14e7: 0x0857, 0x14e8: 0x0863, 0x14e9: 0x0867, + 0x14ea: 0x086b, 0x14eb: 0x0873, 0x14ec: 0x0893, 0x14ed: 0x0897, 0x14ee: 0x089f, 0x14ef: 0x08af, + 0x14f0: 0x08b7, 0x14f1: 0x08bb, 0x14f2: 0x08bb, 0x14f3: 0x08bb, 0x14f4: 0x166a, 0x14f5: 0x0e93, + 0x14f6: 0x08cf, 0x14f7: 0x08d7, 0x14f8: 0x166f, 0x14f9: 0x08e3, 0x14fa: 0x08eb, 0x14fb: 0x08f3, + 0x14fc: 0x091b, 0x14fd: 0x0907, 0x14fe: 0x0913, 0x14ff: 0x0917, + // Block 0x54, offset 0x1500 + 0x1500: 0x091f, 0x1501: 0x0927, 0x1502: 0x092b, 0x1503: 0x0933, 0x1504: 0x093b, 0x1505: 0x093f, + 0x1506: 0x093f, 0x1507: 0x0947, 0x1508: 0x094f, 0x1509: 0x0953, 0x150a: 0x095f, 0x150b: 0x0983, + 0x150c: 0x0967, 0x150d: 0x0987, 0x150e: 0x096b, 0x150f: 0x0973, 0x1510: 0x080b, 0x1511: 0x09cf, + 0x1512: 0x0997, 0x1513: 0x099b, 0x1514: 0x099f, 0x1515: 0x0993, 0x1516: 0x09a7, 0x1517: 0x09a3, + 0x1518: 0x09bb, 0x1519: 0x1674, 0x151a: 0x09d7, 0x151b: 0x09db, 0x151c: 0x09e3, 0x151d: 0x09ef, + 0x151e: 0x09f7, 0x151f: 0x0a13, 0x1520: 0x1679, 0x1521: 0x167e, 0x1522: 0x0a1f, 0x1523: 0x0a23, + 0x1524: 0x0a27, 0x1525: 0x0a1b, 0x1526: 0x0a2f, 0x1527: 0x05c7, 0x1528: 0x05cb, 0x1529: 0x0a37, + 0x152a: 0x0a3f, 0x152b: 0x0a3f, 0x152c: 0x1683, 0x152d: 0x0a5b, 0x152e: 0x0a5f, 0x152f: 0x0a63, + 0x1530: 0x0a6b, 0x1531: 0x1688, 0x1532: 0x0a73, 0x1533: 0x0a77, 0x1534: 0x0b4f, 0x1535: 0x0a7f, + 0x1536: 0x05cf, 0x1537: 0x0a8b, 0x1538: 0x0a9b, 0x1539: 0x0aa7, 0x153a: 0x0aa3, 0x153b: 0x1692, + 0x153c: 0x0aaf, 0x153d: 0x1697, 0x153e: 0x0abb, 0x153f: 0x0ab7, + // Block 0x55, offset 0x1540 + 0x1540: 0x0abf, 0x1541: 0x0acf, 0x1542: 0x0ad3, 0x1543: 0x05d3, 0x1544: 0x0ae3, 0x1545: 0x0aeb, + 0x1546: 0x0aef, 0x1547: 0x0af3, 0x1548: 0x05d7, 0x1549: 0x169c, 0x154a: 0x05db, 0x154b: 0x0b0f, + 0x154c: 0x0b13, 0x154d: 0x0b17, 0x154e: 0x0b1f, 0x154f: 0x1863, 0x1550: 0x0b37, 0x1551: 0x16a6, + 0x1552: 0x16a6, 0x1553: 0x11d7, 0x1554: 0x0b47, 0x1555: 0x0b47, 0x1556: 0x05df, 0x1557: 0x16c9, + 0x1558: 0x179b, 0x1559: 0x0b57, 0x155a: 0x0b5f, 0x155b: 0x05e3, 0x155c: 0x0b73, 0x155d: 0x0b83, + 0x155e: 0x0b87, 0x155f: 0x0b8f, 0x1560: 0x0b9f, 0x1561: 0x05eb, 0x1562: 0x05e7, 0x1563: 0x0ba3, + 0x1564: 0x16ab, 0x1565: 0x0ba7, 0x1566: 0x0bbb, 0x1567: 0x0bbf, 0x1568: 0x0bc3, 0x1569: 0x0bbf, + 0x156a: 0x0bcf, 0x156b: 0x0bd3, 0x156c: 0x0be3, 0x156d: 0x0bdb, 0x156e: 0x0bdf, 0x156f: 0x0be7, + 0x1570: 0x0beb, 0x1571: 0x0bef, 0x1572: 0x0bfb, 0x1573: 0x0bff, 0x1574: 0x0c17, 0x1575: 0x0c1f, + 0x1576: 0x0c2f, 0x1577: 0x0c43, 0x1578: 0x16ba, 0x1579: 0x0c3f, 0x157a: 0x0c33, 0x157b: 0x0c4b, + 0x157c: 0x0c53, 0x157d: 0x0c67, 0x157e: 0x16bf, 0x157f: 0x0c6f, + // Block 0x56, offset 0x1580 + 0x1580: 0x0c63, 0x1581: 0x0c5b, 0x1582: 0x05ef, 0x1583: 0x0c77, 0x1584: 0x0c7f, 0x1585: 0x0c87, + 0x1586: 0x0c7b, 0x1587: 0x05f3, 0x1588: 0x0c97, 0x1589: 0x0c9f, 0x158a: 0x16c4, 0x158b: 0x0ccb, + 0x158c: 0x0cff, 0x158d: 0x0cdb, 0x158e: 0x05ff, 0x158f: 0x0ce7, 0x1590: 0x05fb, 0x1591: 0x05f7, + 0x1592: 0x07c3, 0x1593: 0x07c7, 0x1594: 0x0d03, 0x1595: 0x0ceb, 0x1596: 0x11ab, 0x1597: 0x0663, + 0x1598: 0x0d0f, 0x1599: 0x0d13, 0x159a: 0x0d17, 0x159b: 0x0d2b, 0x159c: 0x0d23, 0x159d: 0x16dd, + 0x159e: 0x0603, 0x159f: 0x0d3f, 0x15a0: 0x0d33, 0x15a1: 0x0d4f, 0x15a2: 0x0d57, 0x15a3: 0x16e7, + 0x15a4: 0x0d5b, 0x15a5: 0x0d47, 0x15a6: 0x0d63, 0x15a7: 0x0607, 0x15a8: 0x0d67, 0x15a9: 0x0d6b, + 0x15aa: 0x0d6f, 0x15ab: 0x0d7b, 0x15ac: 0x16ec, 0x15ad: 0x0d83, 0x15ae: 0x060b, 0x15af: 0x0d8f, + 0x15b0: 0x16f1, 0x15b1: 0x0d93, 0x15b2: 0x060f, 0x15b3: 0x0d9f, 0x15b4: 0x0dab, 0x15b5: 0x0db7, + 0x15b6: 0x0dbb, 0x15b7: 0x16f6, 0x15b8: 0x168d, 0x15b9: 0x16fb, 0x15ba: 0x0ddb, 0x15bb: 0x1700, + 0x15bc: 0x0de7, 0x15bd: 0x0def, 0x15be: 0x0ddf, 0x15bf: 0x0dfb, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x0e0b, 0x15c1: 0x0e1b, 0x15c2: 0x0e0f, 0x15c3: 0x0e13, 0x15c4: 0x0e1f, 0x15c5: 0x0e23, + 0x15c6: 0x1705, 0x15c7: 0x0e07, 0x15c8: 0x0e3b, 0x15c9: 0x0e3f, 0x15ca: 0x0613, 0x15cb: 0x0e53, + 0x15cc: 0x0e4f, 0x15cd: 0x170a, 0x15ce: 0x0e33, 0x15cf: 0x0e6f, 0x15d0: 0x170f, 0x15d1: 0x1714, + 0x15d2: 0x0e73, 0x15d3: 0x0e87, 0x15d4: 0x0e83, 0x15d5: 0x0e7f, 0x15d6: 0x0617, 0x15d7: 0x0e8b, + 0x15d8: 0x0e9b, 0x15d9: 0x0e97, 0x15da: 0x0ea3, 0x15db: 0x1651, 0x15dc: 0x0eb3, 0x15dd: 0x1719, + 0x15de: 0x0ebf, 0x15df: 0x1723, 0x15e0: 0x0ed3, 0x15e1: 0x0edf, 0x15e2: 0x0ef3, 0x15e3: 0x1728, + 0x15e4: 0x0f07, 0x15e5: 0x0f0b, 0x15e6: 0x172d, 0x15e7: 0x1732, 0x15e8: 0x0f27, 0x15e9: 0x0f37, + 0x15ea: 0x061b, 0x15eb: 0x0f3b, 0x15ec: 0x061f, 0x15ed: 0x061f, 0x15ee: 0x0f53, 0x15ef: 0x0f57, + 0x15f0: 0x0f5f, 0x15f1: 0x0f63, 0x15f2: 0x0f6f, 0x15f3: 0x0623, 0x15f4: 0x0f87, 0x15f5: 0x1737, + 0x15f6: 0x0fa3, 0x15f7: 0x173c, 0x15f8: 0x0faf, 0x15f9: 0x16a1, 0x15fa: 0x0fbf, 0x15fb: 0x1741, + 0x15fc: 0x1746, 0x15fd: 0x174b, 0x15fe: 0x0627, 0x15ff: 0x062b, + // Block 0x58, offset 0x1600 + 0x1600: 0x0ff7, 0x1601: 0x1755, 0x1602: 0x1750, 0x1603: 0x175a, 0x1604: 0x175f, 0x1605: 0x0fff, + 0x1606: 0x1003, 0x1607: 0x1003, 0x1608: 0x100b, 0x1609: 0x0633, 0x160a: 0x100f, 0x160b: 0x0637, + 0x160c: 0x063b, 0x160d: 0x1769, 0x160e: 0x1023, 0x160f: 0x102b, 0x1610: 0x1037, 0x1611: 0x063f, + 0x1612: 0x176e, 0x1613: 0x105b, 0x1614: 0x1773, 0x1615: 0x1778, 0x1616: 0x107b, 0x1617: 0x1093, + 0x1618: 0x0643, 0x1619: 0x109b, 0x161a: 0x109f, 0x161b: 0x10a3, 0x161c: 0x177d, 0x161d: 0x1782, + 0x161e: 0x1782, 0x161f: 0x10bb, 0x1620: 0x0647, 0x1621: 0x1787, 0x1622: 0x10cf, 0x1623: 0x10d3, + 0x1624: 0x064b, 0x1625: 0x178c, 0x1626: 0x10ef, 0x1627: 0x064f, 0x1628: 0x10ff, 0x1629: 0x10f7, + 0x162a: 0x1107, 0x162b: 0x1796, 0x162c: 0x111f, 0x162d: 0x0653, 0x162e: 0x112b, 0x162f: 0x1133, + 0x1630: 0x1143, 0x1631: 0x0657, 0x1632: 0x17a0, 0x1633: 0x17a5, 0x1634: 0x065b, 0x1635: 0x17aa, + 0x1636: 0x115b, 0x1637: 0x17af, 0x1638: 0x1167, 0x1639: 0x1173, 0x163a: 0x117b, 0x163b: 0x17b4, + 0x163c: 0x17b9, 0x163d: 0x118f, 0x163e: 0x17be, 0x163f: 0x1197, + // Block 0x59, offset 0x1640 + 0x1640: 0x16ce, 0x1641: 0x065f, 0x1642: 0x11af, 0x1643: 0x11b3, 0x1644: 0x0667, 0x1645: 0x11b7, + 0x1646: 0x0a33, 0x1647: 0x17c3, 0x1648: 0x17c8, 0x1649: 0x16d3, 0x164a: 0x16d8, 0x164b: 0x11d7, + 0x164c: 0x11db, 0x164d: 0x13f3, 0x164e: 0x066b, 0x164f: 0x1207, 0x1650: 0x1203, 0x1651: 0x120b, + 0x1652: 0x083f, 0x1653: 0x120f, 0x1654: 0x1213, 0x1655: 0x1217, 0x1656: 0x121f, 0x1657: 0x17cd, + 0x1658: 0x121b, 0x1659: 0x1223, 0x165a: 0x1237, 0x165b: 0x123b, 0x165c: 0x1227, 0x165d: 0x123f, + 0x165e: 0x1253, 0x165f: 0x1267, 0x1660: 0x1233, 0x1661: 0x1247, 0x1662: 0x124b, 0x1663: 0x124f, + 0x1664: 0x17d2, 0x1665: 0x17dc, 0x1666: 0x17d7, 0x1667: 0x066f, 0x1668: 0x126f, 0x1669: 0x1273, + 0x166a: 0x127b, 0x166b: 0x17f0, 0x166c: 0x127f, 0x166d: 0x17e1, 0x166e: 0x0673, 0x166f: 0x0677, + 0x1670: 0x17e6, 0x1671: 0x17eb, 0x1672: 0x067b, 0x1673: 0x129f, 0x1674: 0x12a3, 0x1675: 0x12a7, + 0x1676: 0x12ab, 0x1677: 0x12b7, 0x1678: 0x12b3, 0x1679: 0x12bf, 0x167a: 0x12bb, 0x167b: 0x12cb, + 0x167c: 0x12c3, 0x167d: 0x12c7, 0x167e: 0x12cf, 0x167f: 0x067f, + // Block 0x5a, offset 0x1680 + 0x1680: 0x12d7, 0x1681: 0x12db, 0x1682: 0x0683, 0x1683: 0x12eb, 0x1684: 0x12ef, 0x1685: 0x17f5, + 0x1686: 0x12fb, 0x1687: 0x12ff, 0x1688: 0x0687, 0x1689: 0x130b, 0x168a: 0x05bb, 0x168b: 0x17fa, + 0x168c: 0x17ff, 0x168d: 0x068b, 0x168e: 0x068f, 0x168f: 0x1337, 0x1690: 0x134f, 0x1691: 0x136b, + 0x1692: 0x137b, 0x1693: 0x1804, 0x1694: 0x138f, 0x1695: 0x1393, 0x1696: 0x13ab, 0x1697: 0x13b7, + 0x1698: 0x180e, 0x1699: 0x1660, 0x169a: 0x13c3, 0x169b: 0x13bf, 0x169c: 0x13cb, 0x169d: 0x1665, + 0x169e: 0x13d7, 0x169f: 0x13e3, 0x16a0: 0x1813, 0x16a1: 0x1818, 0x16a2: 0x1423, 0x16a3: 0x142f, + 0x16a4: 0x1437, 0x16a5: 0x181d, 0x16a6: 0x143b, 0x16a7: 0x1467, 0x16a8: 0x1473, 0x16a9: 0x1477, + 0x16aa: 0x146f, 0x16ab: 0x1483, 0x16ac: 0x1487, 0x16ad: 0x1822, 0x16ae: 0x1493, 0x16af: 0x0693, + 0x16b0: 0x149b, 0x16b1: 0x1827, 0x16b2: 0x0697, 0x16b3: 0x14d3, 0x16b4: 0x0ac3, 0x16b5: 0x14eb, + 0x16b6: 0x182c, 0x16b7: 0x1836, 0x16b8: 0x069b, 0x16b9: 0x069f, 0x16ba: 0x1513, 0x16bb: 0x183b, + 0x16bc: 0x06a3, 0x16bd: 0x1840, 0x16be: 0x152b, 0x16bf: 0x152b, + // Block 0x5b, offset 0x16c0 + 0x16c0: 0x1533, 0x16c1: 0x1845, 0x16c2: 0x154b, 0x16c3: 0x06a7, 0x16c4: 0x155b, 0x16c5: 0x1567, + 0x16c6: 0x156f, 0x16c7: 0x1577, 0x16c8: 0x06ab, 0x16c9: 0x184a, 0x16ca: 0x158b, 0x16cb: 0x15a7, + 0x16cc: 0x15b3, 0x16cd: 0x06af, 0x16ce: 0x06b3, 0x16cf: 0x15b7, 0x16d0: 0x184f, 0x16d1: 0x06b7, + 0x16d2: 0x1854, 0x16d3: 0x1859, 0x16d4: 0x185e, 0x16d5: 0x15db, 0x16d6: 0x06bb, 0x16d7: 0x15ef, + 0x16d8: 0x15f7, 0x16d9: 0x15fb, 0x16da: 0x1603, 0x16db: 0x160b, 0x16dc: 0x1613, 0x16dd: 0x1868, +} + +// nfkcIndex: 22 blocks, 1408 entries, 1408 bytes +// Block 0 is the zero block. +var nfkcIndex = [1408]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x5a, 0xc3: 0x01, 0xc4: 0x02, 0xc5: 0x03, 0xc6: 0x5b, 0xc7: 0x04, + 0xc8: 0x05, 0xca: 0x5c, 0xcb: 0x5d, 0xcc: 0x06, 0xcd: 0x07, 0xce: 0x08, 0xcf: 0x09, + 0xd0: 0x0a, 0xd1: 0x5e, 0xd2: 0x5f, 0xd3: 0x0b, 0xd6: 0x0c, 0xd7: 0x60, + 0xd8: 0x61, 0xd9: 0x0d, 0xdb: 0x62, 0xdc: 0x63, 0xdd: 0x64, 0xdf: 0x65, + 0xe0: 0x02, 0xe1: 0x03, 0xe2: 0x04, 0xe3: 0x05, + 0xea: 0x06, 0xeb: 0x07, 0xec: 0x08, 0xed: 0x09, 0xef: 0x0a, + 0xf0: 0x13, + // Block 0x4, offset 0x100 + 0x120: 0x66, 0x121: 0x67, 0x123: 0x68, 0x124: 0x69, 0x125: 0x6a, 0x126: 0x6b, 0x127: 0x6c, + 0x128: 0x6d, 0x129: 0x6e, 0x12a: 0x6f, 0x12b: 0x70, 0x12c: 0x6b, 0x12d: 0x71, 0x12e: 0x72, 0x12f: 0x73, + 0x131: 0x74, 0x132: 0x75, 0x133: 0x76, 0x134: 0x77, 0x135: 0x78, 0x137: 0x79, + 0x138: 0x7a, 0x139: 0x7b, 0x13a: 0x7c, 0x13b: 0x7d, 0x13c: 0x7e, 0x13d: 0x7f, 0x13e: 0x80, 0x13f: 0x81, + // Block 0x5, offset 0x140 + 0x140: 0x82, 0x142: 0x83, 0x143: 0x84, 0x144: 0x85, 0x145: 0x86, 0x146: 0x87, 0x147: 0x88, + 0x14d: 0x89, + 0x15c: 0x8a, 0x15f: 0x8b, + 0x162: 0x8c, 0x164: 0x8d, + 0x168: 0x8e, 0x169: 0x8f, 0x16a: 0x90, 0x16c: 0x0e, 0x16d: 0x91, 0x16e: 0x92, 0x16f: 0x93, + 0x170: 0x94, 0x173: 0x95, 0x174: 0x96, 0x175: 0x0f, 0x176: 0x10, 0x177: 0x97, + 0x178: 0x11, 0x179: 0x12, 0x17a: 0x13, 0x17b: 0x14, 0x17c: 0x15, 0x17d: 0x16, 0x17e: 0x17, 0x17f: 0x18, + // Block 0x6, offset 0x180 + 0x180: 0x98, 0x181: 0x99, 0x182: 0x9a, 0x183: 0x9b, 0x184: 0x19, 0x185: 0x1a, 0x186: 0x9c, 0x187: 0x9d, + 0x188: 0x9e, 0x189: 0x1b, 0x18a: 0x1c, 0x18b: 0x9f, 0x18c: 0xa0, + 0x191: 0x1d, 0x192: 0x1e, 0x193: 0xa1, + 0x1a8: 0xa2, 0x1a9: 0xa3, 0x1ab: 0xa4, + 0x1b1: 0xa5, 0x1b3: 0xa6, 0x1b5: 0xa7, 0x1b7: 0xa8, + 0x1ba: 0xa9, 0x1bb: 0xaa, 0x1bc: 0x1f, 0x1bd: 0x20, 0x1be: 0x21, 0x1bf: 0xab, + // Block 0x7, offset 0x1c0 + 0x1c0: 0xac, 0x1c1: 0x22, 0x1c2: 0x23, 0x1c3: 0x24, 0x1c4: 0xad, 0x1c5: 0x25, 0x1c6: 0x26, + 0x1c8: 0x27, 0x1c9: 0x28, 0x1ca: 0x29, 0x1cb: 0x2a, 0x1cc: 0x2b, 0x1cd: 0x2c, 0x1ce: 0x2d, 0x1cf: 0x2e, + // Block 0x8, offset 0x200 + 0x219: 0xae, 0x21a: 0xaf, 0x21b: 0xb0, 0x21d: 0xb1, 0x21f: 0xb2, + 0x220: 0xb3, 0x223: 0xb4, 0x224: 0xb5, 0x225: 0xb6, 0x226: 0xb7, 0x227: 0xb8, + 0x22a: 0xb9, 0x22b: 0xba, 0x22d: 0xbb, 0x22f: 0xbc, + 0x230: 0xbd, 0x231: 0xbe, 0x232: 0xbf, 0x233: 0xc0, 0x234: 0xc1, 0x235: 0xc2, 0x236: 0xc3, 0x237: 0xbd, + 0x238: 0xbe, 0x239: 0xbf, 0x23a: 0xc0, 0x23b: 0xc1, 0x23c: 0xc2, 0x23d: 0xc3, 0x23e: 0xbd, 0x23f: 0xbe, + // Block 0x9, offset 0x240 + 0x240: 0xbf, 0x241: 0xc0, 0x242: 0xc1, 0x243: 0xc2, 0x244: 0xc3, 0x245: 0xbd, 0x246: 0xbe, 0x247: 0xbf, + 0x248: 0xc0, 0x249: 0xc1, 0x24a: 0xc2, 0x24b: 0xc3, 0x24c: 0xbd, 0x24d: 0xbe, 0x24e: 0xbf, 0x24f: 0xc0, + 0x250: 0xc1, 0x251: 0xc2, 0x252: 0xc3, 0x253: 0xbd, 0x254: 0xbe, 0x255: 0xbf, 0x256: 0xc0, 0x257: 0xc1, + 0x258: 0xc2, 0x259: 0xc3, 0x25a: 0xbd, 0x25b: 0xbe, 0x25c: 0xbf, 0x25d: 0xc0, 0x25e: 0xc1, 0x25f: 0xc2, + 0x260: 0xc3, 0x261: 0xbd, 0x262: 0xbe, 0x263: 0xbf, 0x264: 0xc0, 0x265: 0xc1, 0x266: 0xc2, 0x267: 0xc3, + 0x268: 0xbd, 0x269: 0xbe, 0x26a: 0xbf, 0x26b: 0xc0, 0x26c: 0xc1, 0x26d: 0xc2, 0x26e: 0xc3, 0x26f: 0xbd, + 0x270: 0xbe, 0x271: 0xbf, 0x272: 0xc0, 0x273: 0xc1, 0x274: 0xc2, 0x275: 0xc3, 0x276: 0xbd, 0x277: 0xbe, + 0x278: 0xbf, 0x279: 0xc0, 0x27a: 0xc1, 0x27b: 0xc2, 0x27c: 0xc3, 0x27d: 0xbd, 0x27e: 0xbe, 0x27f: 0xbf, + // Block 0xa, offset 0x280 + 0x280: 0xc0, 0x281: 0xc1, 0x282: 0xc2, 0x283: 0xc3, 0x284: 0xbd, 0x285: 0xbe, 0x286: 0xbf, 0x287: 0xc0, + 0x288: 0xc1, 0x289: 0xc2, 0x28a: 0xc3, 0x28b: 0xbd, 0x28c: 0xbe, 0x28d: 0xbf, 0x28e: 0xc0, 0x28f: 0xc1, + 0x290: 0xc2, 0x291: 0xc3, 0x292: 0xbd, 0x293: 0xbe, 0x294: 0xbf, 0x295: 0xc0, 0x296: 0xc1, 0x297: 0xc2, + 0x298: 0xc3, 0x299: 0xbd, 0x29a: 0xbe, 0x29b: 0xbf, 0x29c: 0xc0, 0x29d: 0xc1, 0x29e: 0xc2, 0x29f: 0xc3, + 0x2a0: 0xbd, 0x2a1: 0xbe, 0x2a2: 0xbf, 0x2a3: 0xc0, 0x2a4: 0xc1, 0x2a5: 0xc2, 0x2a6: 0xc3, 0x2a7: 0xbd, + 0x2a8: 0xbe, 0x2a9: 0xbf, 0x2aa: 0xc0, 0x2ab: 0xc1, 0x2ac: 0xc2, 0x2ad: 0xc3, 0x2ae: 0xbd, 0x2af: 0xbe, + 0x2b0: 0xbf, 0x2b1: 0xc0, 0x2b2: 0xc1, 0x2b3: 0xc2, 0x2b4: 0xc3, 0x2b5: 0xbd, 0x2b6: 0xbe, 0x2b7: 0xbf, + 0x2b8: 0xc0, 0x2b9: 0xc1, 0x2ba: 0xc2, 0x2bb: 0xc3, 0x2bc: 0xbd, 0x2bd: 0xbe, 0x2be: 0xbf, 0x2bf: 0xc0, + // Block 0xb, offset 0x2c0 + 0x2c0: 0xc1, 0x2c1: 0xc2, 0x2c2: 0xc3, 0x2c3: 0xbd, 0x2c4: 0xbe, 0x2c5: 0xbf, 0x2c6: 0xc0, 0x2c7: 0xc1, + 0x2c8: 0xc2, 0x2c9: 0xc3, 0x2ca: 0xbd, 0x2cb: 0xbe, 0x2cc: 0xbf, 0x2cd: 0xc0, 0x2ce: 0xc1, 0x2cf: 0xc2, + 0x2d0: 0xc3, 0x2d1: 0xbd, 0x2d2: 0xbe, 0x2d3: 0xbf, 0x2d4: 0xc0, 0x2d5: 0xc1, 0x2d6: 0xc2, 0x2d7: 0xc3, + 0x2d8: 0xbd, 0x2d9: 0xbe, 0x2da: 0xbf, 0x2db: 0xc0, 0x2dc: 0xc1, 0x2dd: 0xc2, 0x2de: 0xc4, + // Block 0xc, offset 0x300 + 0x324: 0x2f, 0x325: 0x30, 0x326: 0x31, 0x327: 0x32, + 0x328: 0x33, 0x329: 0x34, 0x32a: 0x35, 0x32b: 0x36, 0x32c: 0x37, 0x32d: 0x38, 0x32e: 0x39, 0x32f: 0x3a, + 0x330: 0x3b, 0x331: 0x3c, 0x332: 0x3d, 0x333: 0x3e, 0x334: 0x3f, 0x335: 0x40, 0x336: 0x41, 0x337: 0x42, + 0x338: 0x43, 0x339: 0x44, 0x33a: 0x45, 0x33b: 0x46, 0x33c: 0xc5, 0x33d: 0x47, 0x33e: 0x48, 0x33f: 0x49, + // Block 0xd, offset 0x340 + 0x347: 0xc6, + 0x34b: 0xc7, 0x34d: 0xc8, + 0x368: 0xc9, 0x36b: 0xca, + // Block 0xe, offset 0x380 + 0x381: 0xcb, 0x382: 0xcc, 0x384: 0xcd, 0x385: 0xb7, 0x387: 0xce, + 0x388: 0xcf, 0x38b: 0xd0, 0x38c: 0x6b, 0x38d: 0xd1, + 0x391: 0xd2, 0x392: 0xd3, 0x393: 0xd4, 0x396: 0xd5, 0x397: 0xd6, + 0x398: 0xd7, 0x39a: 0xd8, 0x39c: 0xd9, + 0x3b0: 0xd7, + // Block 0xf, offset 0x3c0 + 0x3eb: 0xda, 0x3ec: 0xdb, + // Block 0x10, offset 0x400 + 0x432: 0xdc, + // Block 0x11, offset 0x440 + 0x445: 0xdd, 0x446: 0xde, 0x447: 0xdf, + 0x449: 0xe0, + 0x450: 0xe1, 0x451: 0xe2, 0x452: 0xe3, 0x453: 0xe4, 0x454: 0xe5, 0x455: 0xe6, 0x456: 0xe7, 0x457: 0xe8, + 0x458: 0xe9, 0x459: 0xea, 0x45a: 0x4a, 0x45b: 0xeb, 0x45c: 0xec, 0x45d: 0xed, 0x45e: 0xee, 0x45f: 0x4b, + // Block 0x12, offset 0x480 + 0x480: 0xef, + 0x4a3: 0xf0, 0x4a5: 0xf1, + 0x4b8: 0x4c, 0x4b9: 0x4d, 0x4ba: 0x4e, + // Block 0x13, offset 0x4c0 + 0x4c4: 0x4f, 0x4c5: 0xf2, 0x4c6: 0xf3, + 0x4c8: 0x50, 0x4c9: 0xf4, + // Block 0x14, offset 0x500 + 0x520: 0x51, 0x521: 0x52, 0x522: 0x53, 0x523: 0x54, 0x524: 0x55, 0x525: 0x56, 0x526: 0x57, 0x527: 0x58, + 0x528: 0x59, + // Block 0x15, offset 0x540 + 0x550: 0x0b, 0x551: 0x0c, 0x556: 0x0d, + 0x55b: 0x0e, 0x55d: 0x0f, 0x55e: 0x10, 0x55f: 0x11, + 0x56f: 0x12, +} + +// nfkcSparseOffset: 155 entries, 310 bytes +var nfkcSparseOffset = []uint16{0x0, 0xe, 0x12, 0x1b, 0x25, 0x35, 0x37, 0x3c, 0x47, 0x56, 0x63, 0x6b, 0x6f, 0x74, 0x76, 0x87, 0x8f, 0x96, 0x99, 0xa0, 0xa4, 0xa8, 0xaa, 0xac, 0xb5, 0xb9, 0xc0, 0xc5, 0xc8, 0xd2, 0xd4, 0xdb, 0xe3, 0xe7, 0xe9, 0xec, 0xf0, 0xf6, 0x107, 0x113, 0x115, 0x11b, 0x11d, 0x11f, 0x121, 0x123, 0x125, 0x127, 0x129, 0x12c, 0x12f, 0x131, 0x134, 0x137, 0x13b, 0x140, 0x149, 0x14b, 0x14e, 0x150, 0x15b, 0x166, 0x176, 0x184, 0x192, 0x1a2, 0x1b0, 0x1b7, 0x1bd, 0x1cc, 0x1d0, 0x1d2, 0x1d6, 0x1d8, 0x1db, 0x1dd, 0x1e0, 0x1e2, 0x1e5, 0x1e7, 0x1e9, 0x1eb, 0x1f7, 0x201, 0x20b, 0x20e, 0x212, 0x214, 0x216, 0x218, 0x21a, 0x21d, 0x21f, 0x221, 0x223, 0x225, 0x22b, 0x22e, 0x232, 0x234, 0x23b, 0x241, 0x247, 0x24f, 0x255, 0x25b, 0x261, 0x265, 0x267, 0x269, 0x26b, 0x26d, 0x273, 0x276, 0x279, 0x281, 0x288, 0x28b, 0x28e, 0x290, 0x298, 0x29b, 0x2a2, 0x2a5, 0x2ab, 0x2ad, 0x2af, 0x2b2, 0x2b4, 0x2b6, 0x2b8, 0x2ba, 0x2c7, 0x2d1, 0x2d3, 0x2d5, 0x2d9, 0x2de, 0x2ea, 0x2ef, 0x2f8, 0x2fe, 0x303, 0x307, 0x30c, 0x310, 0x320, 0x32e, 0x33c, 0x34a, 0x350, 0x352, 0x355, 0x35f, 0x361} + +// nfkcSparseValues: 875 entries, 3500 bytes +var nfkcSparseValues = [875]valueRange{ + // Block 0x0, offset 0x0 + {value: 0x0002, lo: 0x0d}, + {value: 0x0001, lo: 0xa0, hi: 0xa0}, + {value: 0x4278, lo: 0xa8, hi: 0xa8}, + {value: 0x0083, lo: 0xaa, hi: 0xaa}, + {value: 0x4264, lo: 0xaf, hi: 0xaf}, + {value: 0x0025, lo: 0xb2, hi: 0xb3}, + {value: 0x425a, lo: 0xb4, hi: 0xb4}, + {value: 0x01dc, lo: 0xb5, hi: 0xb5}, + {value: 0x4291, lo: 0xb8, hi: 0xb8}, + {value: 0x0023, lo: 0xb9, hi: 0xb9}, + {value: 0x009f, lo: 0xba, hi: 0xba}, + {value: 0x221c, lo: 0xbc, hi: 0xbc}, + {value: 0x2210, lo: 0xbd, hi: 0xbd}, + {value: 0x22b2, lo: 0xbe, hi: 0xbe}, + // Block 0x1, offset 0xe + {value: 0x0091, lo: 0x03}, + {value: 0x4778, lo: 0xa0, hi: 0xa1}, + {value: 0x47aa, lo: 0xaf, hi: 0xb0}, + {value: 0xa000, lo: 0xb7, hi: 0xb7}, + // Block 0x2, offset 0x12 + {value: 0x0003, lo: 0x08}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x0091, lo: 0xb0, hi: 0xb0}, + {value: 0x0119, lo: 0xb1, hi: 0xb1}, + {value: 0x0095, lo: 0xb2, hi: 0xb2}, + {value: 0x00a5, lo: 0xb3, hi: 0xb3}, + {value: 0x0143, lo: 0xb4, hi: 0xb6}, + {value: 0x00af, lo: 0xb7, hi: 0xb7}, + {value: 0x00b3, lo: 0xb8, hi: 0xb8}, + // Block 0x3, offset 0x1b + {value: 0x000a, lo: 0x09}, + {value: 0x426e, lo: 0x98, hi: 0x98}, + {value: 0x4273, lo: 0x99, hi: 0x9a}, + {value: 0x4296, lo: 0x9b, hi: 0x9b}, + {value: 0x425f, lo: 0x9c, hi: 0x9c}, + {value: 0x4282, lo: 0x9d, hi: 0x9d}, + {value: 0x0113, lo: 0xa0, hi: 0xa0}, + {value: 0x0099, lo: 0xa1, hi: 0xa1}, + {value: 0x00a7, lo: 0xa2, hi: 0xa3}, + {value: 0x0167, lo: 0xa4, hi: 0xa4}, + // Block 0x4, offset 0x25 + {value: 0x0000, lo: 0x0f}, + {value: 0xa000, lo: 0x83, hi: 0x83}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0xa000, lo: 0x8b, hi: 0x8b}, + {value: 0xa000, lo: 0x8d, hi: 0x8d}, + {value: 0x37a5, lo: 0x90, hi: 0x90}, + {value: 0x37b1, lo: 0x91, hi: 0x91}, + {value: 0x379f, lo: 0x93, hi: 0x93}, + {value: 0xa000, lo: 0x96, hi: 0x96}, + {value: 0x3817, lo: 0x97, hi: 0x97}, + {value: 0x37e1, lo: 0x9c, hi: 0x9c}, + {value: 0x37c9, lo: 0x9d, hi: 0x9d}, + {value: 0x37f3, lo: 0x9e, hi: 0x9e}, + {value: 0xa000, lo: 0xb4, hi: 0xb5}, + {value: 0x381d, lo: 0xb6, hi: 0xb6}, + {value: 0x3823, lo: 0xb7, hi: 0xb7}, + // Block 0x5, offset 0x35 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x83, hi: 0x87}, + // Block 0x6, offset 0x37 + {value: 0x0001, lo: 0x04}, + {value: 0x8113, lo: 0x81, hi: 0x82}, + {value: 0x8132, lo: 0x84, hi: 0x84}, + {value: 0x812d, lo: 0x85, hi: 0x85}, + {value: 0x810d, lo: 0x87, hi: 0x87}, + // Block 0x7, offset 0x3c + {value: 0x0000, lo: 0x0a}, + {value: 0x8132, lo: 0x90, hi: 0x97}, + {value: 0x8119, lo: 0x98, hi: 0x98}, + {value: 0x811a, lo: 0x99, hi: 0x99}, + {value: 0x811b, lo: 0x9a, hi: 0x9a}, + {value: 0x3841, lo: 0xa2, hi: 0xa2}, + {value: 0x3847, lo: 0xa3, hi: 0xa3}, + {value: 0x3853, lo: 0xa4, hi: 0xa4}, + {value: 0x384d, lo: 0xa5, hi: 0xa5}, + {value: 0x3859, lo: 0xa6, hi: 0xa6}, + {value: 0xa000, lo: 0xa7, hi: 0xa7}, + // Block 0x8, offset 0x47 + {value: 0x0000, lo: 0x0e}, + {value: 0x386b, lo: 0x80, hi: 0x80}, + {value: 0xa000, lo: 0x81, hi: 0x81}, + {value: 0x385f, lo: 0x82, hi: 0x82}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x3865, lo: 0x93, hi: 0x93}, + {value: 0xa000, lo: 0x95, hi: 0x95}, + {value: 0x8132, lo: 0x96, hi: 0x9c}, + {value: 0x8132, lo: 0x9f, hi: 0xa2}, + {value: 0x812d, lo: 0xa3, hi: 0xa3}, + {value: 0x8132, lo: 0xa4, hi: 0xa4}, + {value: 0x8132, lo: 0xa7, hi: 0xa8}, + {value: 0x812d, lo: 0xaa, hi: 0xaa}, + {value: 0x8132, lo: 0xab, hi: 0xac}, + {value: 0x812d, lo: 0xad, hi: 0xad}, + // Block 0x9, offset 0x56 + {value: 0x0000, lo: 0x0c}, + {value: 0x811f, lo: 0x91, hi: 0x91}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + {value: 0x812d, lo: 0xb1, hi: 0xb1}, + {value: 0x8132, lo: 0xb2, hi: 0xb3}, + {value: 0x812d, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb5, hi: 0xb6}, + {value: 0x812d, lo: 0xb7, hi: 0xb9}, + {value: 0x8132, lo: 0xba, hi: 0xba}, + {value: 0x812d, lo: 0xbb, hi: 0xbc}, + {value: 0x8132, lo: 0xbd, hi: 0xbd}, + {value: 0x812d, lo: 0xbe, hi: 0xbe}, + {value: 0x8132, lo: 0xbf, hi: 0xbf}, + // Block 0xa, offset 0x63 + {value: 0x0005, lo: 0x07}, + {value: 0x8132, lo: 0x80, hi: 0x80}, + {value: 0x8132, lo: 0x81, hi: 0x81}, + {value: 0x812d, lo: 0x82, hi: 0x83}, + {value: 0x812d, lo: 0x84, hi: 0x85}, + {value: 0x812d, lo: 0x86, hi: 0x87}, + {value: 0x812d, lo: 0x88, hi: 0x89}, + {value: 0x8132, lo: 0x8a, hi: 0x8a}, + // Block 0xb, offset 0x6b + {value: 0x0000, lo: 0x03}, + {value: 0x8132, lo: 0xab, hi: 0xb1}, + {value: 0x812d, lo: 0xb2, hi: 0xb2}, + {value: 0x8132, lo: 0xb3, hi: 0xb3}, + // Block 0xc, offset 0x6f + {value: 0x0000, lo: 0x04}, + {value: 0x8132, lo: 0x96, hi: 0x99}, + {value: 0x8132, lo: 0x9b, hi: 0xa3}, + {value: 0x8132, lo: 0xa5, hi: 0xa7}, + {value: 0x8132, lo: 0xa9, hi: 0xad}, + // Block 0xd, offset 0x74 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x99, hi: 0x9b}, + // Block 0xe, offset 0x76 + {value: 0x0000, lo: 0x10}, + {value: 0x8132, lo: 0x94, hi: 0xa1}, + {value: 0x812d, lo: 0xa3, hi: 0xa3}, + {value: 0x8132, lo: 0xa4, hi: 0xa5}, + {value: 0x812d, lo: 0xa6, hi: 0xa6}, + {value: 0x8132, lo: 0xa7, hi: 0xa8}, + {value: 0x812d, lo: 0xa9, hi: 0xa9}, + {value: 0x8132, lo: 0xaa, hi: 0xac}, + {value: 0x812d, lo: 0xad, hi: 0xaf}, + {value: 0x8116, lo: 0xb0, hi: 0xb0}, + {value: 0x8117, lo: 0xb1, hi: 0xb1}, + {value: 0x8118, lo: 0xb2, hi: 0xb2}, + {value: 0x8132, lo: 0xb3, hi: 0xb5}, + {value: 0x812d, lo: 0xb6, hi: 0xb6}, + {value: 0x8132, lo: 0xb7, hi: 0xb8}, + {value: 0x812d, lo: 0xb9, hi: 0xba}, + {value: 0x8132, lo: 0xbb, hi: 0xbf}, + // Block 0xf, offset 0x87 + {value: 0x0000, lo: 0x07}, + {value: 0xa000, lo: 0xa8, hi: 0xa8}, + {value: 0x3ed8, lo: 0xa9, hi: 0xa9}, + {value: 0xa000, lo: 0xb0, hi: 0xb0}, + {value: 0x3ee0, lo: 0xb1, hi: 0xb1}, + {value: 0xa000, lo: 0xb3, hi: 0xb3}, + {value: 0x3ee8, lo: 0xb4, hi: 0xb4}, + {value: 0x9902, lo: 0xbc, hi: 0xbc}, + // Block 0x10, offset 0x8f + {value: 0x0008, lo: 0x06}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x8132, lo: 0x91, hi: 0x91}, + {value: 0x812d, lo: 0x92, hi: 0x92}, + {value: 0x8132, lo: 0x93, hi: 0x93}, + {value: 0x8132, lo: 0x94, hi: 0x94}, + {value: 0x45b2, lo: 0x98, hi: 0x9f}, + // Block 0x11, offset 0x96 + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x12, offset 0x99 + {value: 0x0008, lo: 0x06}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2c9e, lo: 0x8b, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + {value: 0x45f2, lo: 0x9c, hi: 0x9d}, + {value: 0x4602, lo: 0x9f, hi: 0x9f}, + // Block 0x13, offset 0xa0 + {value: 0x0000, lo: 0x03}, + {value: 0x462a, lo: 0xb3, hi: 0xb3}, + {value: 0x4632, lo: 0xb6, hi: 0xb6}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + // Block 0x14, offset 0xa4 + {value: 0x0008, lo: 0x03}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x460a, lo: 0x99, hi: 0x9b}, + {value: 0x4622, lo: 0x9e, hi: 0x9e}, + // Block 0x15, offset 0xa8 + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + // Block 0x16, offset 0xaa + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + // Block 0x17, offset 0xac + {value: 0x0000, lo: 0x08}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2cb6, lo: 0x88, hi: 0x88}, + {value: 0x2cae, lo: 0x8b, hi: 0x8b}, + {value: 0x2cbe, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x96, hi: 0x97}, + {value: 0x463a, lo: 0x9c, hi: 0x9c}, + {value: 0x4642, lo: 0x9d, hi: 0x9d}, + // Block 0x18, offset 0xb5 + {value: 0x0000, lo: 0x03}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x2cc6, lo: 0x94, hi: 0x94}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x19, offset 0xb9 + {value: 0x0000, lo: 0x06}, + {value: 0xa000, lo: 0x86, hi: 0x87}, + {value: 0x2cce, lo: 0x8a, hi: 0x8a}, + {value: 0x2cde, lo: 0x8b, hi: 0x8b}, + {value: 0x2cd6, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + // Block 0x1a, offset 0xc0 + {value: 0x1801, lo: 0x04}, + {value: 0xa000, lo: 0x86, hi: 0x86}, + {value: 0x3ef0, lo: 0x88, hi: 0x88}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x8120, lo: 0x95, hi: 0x96}, + // Block 0x1b, offset 0xc5 + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + {value: 0xa000, lo: 0xbf, hi: 0xbf}, + // Block 0x1c, offset 0xc8 + {value: 0x0000, lo: 0x09}, + {value: 0x2ce6, lo: 0x80, hi: 0x80}, + {value: 0x9900, lo: 0x82, hi: 0x82}, + {value: 0xa000, lo: 0x86, hi: 0x86}, + {value: 0x2cee, lo: 0x87, hi: 0x87}, + {value: 0x2cf6, lo: 0x88, hi: 0x88}, + {value: 0x2f50, lo: 0x8a, hi: 0x8a}, + {value: 0x2dd8, lo: 0x8b, hi: 0x8b}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x95, hi: 0x96}, + // Block 0x1d, offset 0xd2 + {value: 0x0000, lo: 0x01}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x1e, offset 0xd4 + {value: 0x0000, lo: 0x06}, + {value: 0xa000, lo: 0x86, hi: 0x87}, + {value: 0x2cfe, lo: 0x8a, hi: 0x8a}, + {value: 0x2d0e, lo: 0x8b, hi: 0x8b}, + {value: 0x2d06, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + // Block 0x1f, offset 0xdb + {value: 0x6bea, lo: 0x07}, + {value: 0x9904, lo: 0x8a, hi: 0x8a}, + {value: 0x9900, lo: 0x8f, hi: 0x8f}, + {value: 0xa000, lo: 0x99, hi: 0x99}, + {value: 0x3ef8, lo: 0x9a, hi: 0x9a}, + {value: 0x2f58, lo: 0x9c, hi: 0x9c}, + {value: 0x2de3, lo: 0x9d, hi: 0x9d}, + {value: 0x2d16, lo: 0x9e, hi: 0x9f}, + // Block 0x20, offset 0xe3 + {value: 0x0000, lo: 0x03}, + {value: 0x2621, lo: 0xb3, hi: 0xb3}, + {value: 0x8122, lo: 0xb8, hi: 0xb9}, + {value: 0x8104, lo: 0xba, hi: 0xba}, + // Block 0x21, offset 0xe7 + {value: 0x0000, lo: 0x01}, + {value: 0x8123, lo: 0x88, hi: 0x8b}, + // Block 0x22, offset 0xe9 + {value: 0x0000, lo: 0x02}, + {value: 0x2636, lo: 0xb3, hi: 0xb3}, + {value: 0x8124, lo: 0xb8, hi: 0xb9}, + // Block 0x23, offset 0xec + {value: 0x0000, lo: 0x03}, + {value: 0x8125, lo: 0x88, hi: 0x8b}, + {value: 0x2628, lo: 0x9c, hi: 0x9c}, + {value: 0x262f, lo: 0x9d, hi: 0x9d}, + // Block 0x24, offset 0xf0 + {value: 0x0000, lo: 0x05}, + {value: 0x030b, lo: 0x8c, hi: 0x8c}, + {value: 0x812d, lo: 0x98, hi: 0x99}, + {value: 0x812d, lo: 0xb5, hi: 0xb5}, + {value: 0x812d, lo: 0xb7, hi: 0xb7}, + {value: 0x812b, lo: 0xb9, hi: 0xb9}, + // Block 0x25, offset 0xf6 + {value: 0x0000, lo: 0x10}, + {value: 0x2644, lo: 0x83, hi: 0x83}, + {value: 0x264b, lo: 0x8d, hi: 0x8d}, + {value: 0x2652, lo: 0x92, hi: 0x92}, + {value: 0x2659, lo: 0x97, hi: 0x97}, + {value: 0x2660, lo: 0x9c, hi: 0x9c}, + {value: 0x263d, lo: 0xa9, hi: 0xa9}, + {value: 0x8126, lo: 0xb1, hi: 0xb1}, + {value: 0x8127, lo: 0xb2, hi: 0xb2}, + {value: 0x4a66, lo: 0xb3, hi: 0xb3}, + {value: 0x8128, lo: 0xb4, hi: 0xb4}, + {value: 0x4a6f, lo: 0xb5, hi: 0xb5}, + {value: 0x464a, lo: 0xb6, hi: 0xb6}, + {value: 0x468a, lo: 0xb7, hi: 0xb7}, + {value: 0x4652, lo: 0xb8, hi: 0xb8}, + {value: 0x4695, lo: 0xb9, hi: 0xb9}, + {value: 0x8127, lo: 0xba, hi: 0xbd}, + // Block 0x26, offset 0x107 + {value: 0x0000, lo: 0x0b}, + {value: 0x8127, lo: 0x80, hi: 0x80}, + {value: 0x4a78, lo: 0x81, hi: 0x81}, + {value: 0x8132, lo: 0x82, hi: 0x83}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0x86, hi: 0x87}, + {value: 0x266e, lo: 0x93, hi: 0x93}, + {value: 0x2675, lo: 0x9d, hi: 0x9d}, + {value: 0x267c, lo: 0xa2, hi: 0xa2}, + {value: 0x2683, lo: 0xa7, hi: 0xa7}, + {value: 0x268a, lo: 0xac, hi: 0xac}, + {value: 0x2667, lo: 0xb9, hi: 0xb9}, + // Block 0x27, offset 0x113 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x86, hi: 0x86}, + // Block 0x28, offset 0x115 + {value: 0x0000, lo: 0x05}, + {value: 0xa000, lo: 0xa5, hi: 0xa5}, + {value: 0x2d1e, lo: 0xa6, hi: 0xa6}, + {value: 0x9900, lo: 0xae, hi: 0xae}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + {value: 0x8104, lo: 0xb9, hi: 0xba}, + // Block 0x29, offset 0x11b + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x8d, hi: 0x8d}, + // Block 0x2a, offset 0x11d + {value: 0x0000, lo: 0x01}, + {value: 0x030f, lo: 0xbc, hi: 0xbc}, + // Block 0x2b, offset 0x11f + {value: 0x0000, lo: 0x01}, + {value: 0xa000, lo: 0x80, hi: 0x92}, + // Block 0x2c, offset 0x121 + {value: 0x0000, lo: 0x01}, + {value: 0xb900, lo: 0xa1, hi: 0xb5}, + // Block 0x2d, offset 0x123 + {value: 0x0000, lo: 0x01}, + {value: 0x9900, lo: 0xa8, hi: 0xbf}, + // Block 0x2e, offset 0x125 + {value: 0x0000, lo: 0x01}, + {value: 0x9900, lo: 0x80, hi: 0x82}, + // Block 0x2f, offset 0x127 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x9d, hi: 0x9f}, + // Block 0x30, offset 0x129 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x94, hi: 0x94}, + {value: 0x8104, lo: 0xb4, hi: 0xb4}, + // Block 0x31, offset 0x12c + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x92, hi: 0x92}, + {value: 0x8132, lo: 0x9d, hi: 0x9d}, + // Block 0x32, offset 0x12f + {value: 0x0000, lo: 0x01}, + {value: 0x8131, lo: 0xa9, hi: 0xa9}, + // Block 0x33, offset 0x131 + {value: 0x0004, lo: 0x02}, + {value: 0x812e, lo: 0xb9, hi: 0xba}, + {value: 0x812d, lo: 0xbb, hi: 0xbb}, + // Block 0x34, offset 0x134 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x97, hi: 0x97}, + {value: 0x812d, lo: 0x98, hi: 0x98}, + // Block 0x35, offset 0x137 + {value: 0x0000, lo: 0x03}, + {value: 0x8104, lo: 0xa0, hi: 0xa0}, + {value: 0x8132, lo: 0xb5, hi: 0xbc}, + {value: 0x812d, lo: 0xbf, hi: 0xbf}, + // Block 0x36, offset 0x13b + {value: 0x0000, lo: 0x04}, + {value: 0x8132, lo: 0xb0, hi: 0xb4}, + {value: 0x812d, lo: 0xb5, hi: 0xba}, + {value: 0x8132, lo: 0xbb, hi: 0xbc}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + // Block 0x37, offset 0x140 + {value: 0x0000, lo: 0x08}, + {value: 0x2d66, lo: 0x80, hi: 0x80}, + {value: 0x2d6e, lo: 0x81, hi: 0x81}, + {value: 0xa000, lo: 0x82, hi: 0x82}, + {value: 0x2d76, lo: 0x83, hi: 0x83}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0xab, hi: 0xab}, + {value: 0x812d, lo: 0xac, hi: 0xac}, + {value: 0x8132, lo: 0xad, hi: 0xb3}, + // Block 0x38, offset 0x149 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xaa, hi: 0xab}, + // Block 0x39, offset 0x14b + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xa6, hi: 0xa6}, + {value: 0x8104, lo: 0xb2, hi: 0xb3}, + // Block 0x3a, offset 0x14e + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + // Block 0x3b, offset 0x150 + {value: 0x0000, lo: 0x0a}, + {value: 0x8132, lo: 0x90, hi: 0x92}, + {value: 0x8101, lo: 0x94, hi: 0x94}, + {value: 0x812d, lo: 0x95, hi: 0x99}, + {value: 0x8132, lo: 0x9a, hi: 0x9b}, + {value: 0x812d, lo: 0x9c, hi: 0x9f}, + {value: 0x8132, lo: 0xa0, hi: 0xa0}, + {value: 0x8101, lo: 0xa2, hi: 0xa8}, + {value: 0x812d, lo: 0xad, hi: 0xad}, + {value: 0x8132, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb8, hi: 0xb9}, + // Block 0x3c, offset 0x15b + {value: 0x0002, lo: 0x0a}, + {value: 0x0043, lo: 0xac, hi: 0xac}, + {value: 0x00d1, lo: 0xad, hi: 0xad}, + {value: 0x0045, lo: 0xae, hi: 0xae}, + {value: 0x0049, lo: 0xb0, hi: 0xb1}, + {value: 0x00e6, lo: 0xb2, hi: 0xb2}, + {value: 0x004f, lo: 0xb3, hi: 0xba}, + {value: 0x005f, lo: 0xbc, hi: 0xbc}, + {value: 0x00ef, lo: 0xbd, hi: 0xbd}, + {value: 0x0061, lo: 0xbe, hi: 0xbe}, + {value: 0x0065, lo: 0xbf, hi: 0xbf}, + // Block 0x3d, offset 0x166 + {value: 0x0000, lo: 0x0f}, + {value: 0x8132, lo: 0x80, hi: 0x81}, + {value: 0x812d, lo: 0x82, hi: 0x82}, + {value: 0x8132, lo: 0x83, hi: 0x89}, + {value: 0x812d, lo: 0x8a, hi: 0x8a}, + {value: 0x8132, lo: 0x8b, hi: 0x8c}, + {value: 0x8135, lo: 0x8d, hi: 0x8d}, + {value: 0x812a, lo: 0x8e, hi: 0x8e}, + {value: 0x812d, lo: 0x8f, hi: 0x8f}, + {value: 0x8129, lo: 0x90, hi: 0x90}, + {value: 0x8132, lo: 0x91, hi: 0xb5}, + {value: 0x8132, lo: 0xbb, hi: 0xbb}, + {value: 0x8134, lo: 0xbc, hi: 0xbc}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + {value: 0x8132, lo: 0xbe, hi: 0xbe}, + {value: 0x812d, lo: 0xbf, hi: 0xbf}, + // Block 0x3e, offset 0x176 + {value: 0x0000, lo: 0x0d}, + {value: 0x0001, lo: 0x80, hi: 0x8a}, + {value: 0x043b, lo: 0x91, hi: 0x91}, + {value: 0x429b, lo: 0x97, hi: 0x97}, + {value: 0x001d, lo: 0xa4, hi: 0xa4}, + {value: 0x1873, lo: 0xa5, hi: 0xa5}, + {value: 0x1b5c, lo: 0xa6, hi: 0xa6}, + {value: 0x0001, lo: 0xaf, hi: 0xaf}, + {value: 0x2691, lo: 0xb3, hi: 0xb3}, + {value: 0x27fe, lo: 0xb4, hi: 0xb4}, + {value: 0x2698, lo: 0xb6, hi: 0xb6}, + {value: 0x2808, lo: 0xb7, hi: 0xb7}, + {value: 0x186d, lo: 0xbc, hi: 0xbc}, + {value: 0x4269, lo: 0xbe, hi: 0xbe}, + // Block 0x3f, offset 0x184 + {value: 0x0002, lo: 0x0d}, + {value: 0x1933, lo: 0x87, hi: 0x87}, + {value: 0x1930, lo: 0x88, hi: 0x88}, + {value: 0x1870, lo: 0x89, hi: 0x89}, + {value: 0x298e, lo: 0x97, hi: 0x97}, + {value: 0x0001, lo: 0x9f, hi: 0x9f}, + {value: 0x0021, lo: 0xb0, hi: 0xb0}, + {value: 0x0093, lo: 0xb1, hi: 0xb1}, + {value: 0x0029, lo: 0xb4, hi: 0xb9}, + {value: 0x0017, lo: 0xba, hi: 0xba}, + {value: 0x0467, lo: 0xbb, hi: 0xbb}, + {value: 0x003b, lo: 0xbc, hi: 0xbc}, + {value: 0x0011, lo: 0xbd, hi: 0xbe}, + {value: 0x009d, lo: 0xbf, hi: 0xbf}, + // Block 0x40, offset 0x192 + {value: 0x0002, lo: 0x0f}, + {value: 0x0021, lo: 0x80, hi: 0x89}, + {value: 0x0017, lo: 0x8a, hi: 0x8a}, + {value: 0x0467, lo: 0x8b, hi: 0x8b}, + {value: 0x003b, lo: 0x8c, hi: 0x8c}, + {value: 0x0011, lo: 0x8d, hi: 0x8e}, + {value: 0x0083, lo: 0x90, hi: 0x90}, + {value: 0x008b, lo: 0x91, hi: 0x91}, + {value: 0x009f, lo: 0x92, hi: 0x92}, + {value: 0x00b1, lo: 0x93, hi: 0x93}, + {value: 0x0104, lo: 0x94, hi: 0x94}, + {value: 0x0091, lo: 0x95, hi: 0x95}, + {value: 0x0097, lo: 0x96, hi: 0x99}, + {value: 0x00a1, lo: 0x9a, hi: 0x9a}, + {value: 0x00a7, lo: 0x9b, hi: 0x9c}, + {value: 0x1999, lo: 0xa8, hi: 0xa8}, + // Block 0x41, offset 0x1a2 + {value: 0x0000, lo: 0x0d}, + {value: 0x8132, lo: 0x90, hi: 0x91}, + {value: 0x8101, lo: 0x92, hi: 0x93}, + {value: 0x8132, lo: 0x94, hi: 0x97}, + {value: 0x8101, lo: 0x98, hi: 0x9a}, + {value: 0x8132, lo: 0x9b, hi: 0x9c}, + {value: 0x8132, lo: 0xa1, hi: 0xa1}, + {value: 0x8101, lo: 0xa5, hi: 0xa6}, + {value: 0x8132, lo: 0xa7, hi: 0xa7}, + {value: 0x812d, lo: 0xa8, hi: 0xa8}, + {value: 0x8132, lo: 0xa9, hi: 0xa9}, + {value: 0x8101, lo: 0xaa, hi: 0xab}, + {value: 0x812d, lo: 0xac, hi: 0xaf}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + // Block 0x42, offset 0x1b0 + {value: 0x0007, lo: 0x06}, + {value: 0x2180, lo: 0x89, hi: 0x89}, + {value: 0xa000, lo: 0x90, hi: 0x90}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0xa000, lo: 0x94, hi: 0x94}, + {value: 0x3bb9, lo: 0x9a, hi: 0x9b}, + {value: 0x3bc7, lo: 0xae, hi: 0xae}, + // Block 0x43, offset 0x1b7 + {value: 0x000e, lo: 0x05}, + {value: 0x3bce, lo: 0x8d, hi: 0x8e}, + {value: 0x3bd5, lo: 0x8f, hi: 0x8f}, + {value: 0xa000, lo: 0x90, hi: 0x90}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0xa000, lo: 0x94, hi: 0x94}, + // Block 0x44, offset 0x1bd + {value: 0x0173, lo: 0x0e}, + {value: 0xa000, lo: 0x83, hi: 0x83}, + {value: 0x3be3, lo: 0x84, hi: 0x84}, + {value: 0xa000, lo: 0x88, hi: 0x88}, + {value: 0x3bea, lo: 0x89, hi: 0x89}, + {value: 0xa000, lo: 0x8b, hi: 0x8b}, + {value: 0x3bf1, lo: 0x8c, hi: 0x8c}, + {value: 0xa000, lo: 0xa3, hi: 0xa3}, + {value: 0x3bf8, lo: 0xa4, hi: 0xa4}, + {value: 0xa000, lo: 0xa5, hi: 0xa5}, + {value: 0x3bff, lo: 0xa6, hi: 0xa6}, + {value: 0x269f, lo: 0xac, hi: 0xad}, + {value: 0x26a6, lo: 0xaf, hi: 0xaf}, + {value: 0x281c, lo: 0xb0, hi: 0xb0}, + {value: 0xa000, lo: 0xbc, hi: 0xbc}, + // Block 0x45, offset 0x1cc + {value: 0x0007, lo: 0x03}, + {value: 0x3c68, lo: 0xa0, hi: 0xa1}, + {value: 0x3c92, lo: 0xa2, hi: 0xa3}, + {value: 0x3cbc, lo: 0xaa, hi: 0xad}, + // Block 0x46, offset 0x1d0 + {value: 0x0004, lo: 0x01}, + {value: 0x048b, lo: 0xa9, hi: 0xaa}, + // Block 0x47, offset 0x1d2 + {value: 0x0002, lo: 0x03}, + {value: 0x0057, lo: 0x80, hi: 0x8f}, + {value: 0x0083, lo: 0x90, hi: 0xa9}, + {value: 0x0021, lo: 0xaa, hi: 0xaa}, + // Block 0x48, offset 0x1d6 + {value: 0x0000, lo: 0x01}, + {value: 0x299b, lo: 0x8c, hi: 0x8c}, + // Block 0x49, offset 0x1d8 + {value: 0x0263, lo: 0x02}, + {value: 0x1b8c, lo: 0xb4, hi: 0xb4}, + {value: 0x192d, lo: 0xb5, hi: 0xb6}, + // Block 0x4a, offset 0x1db + {value: 0x0000, lo: 0x01}, + {value: 0x4573, lo: 0x9c, hi: 0x9c}, + // Block 0x4b, offset 0x1dd + {value: 0x0000, lo: 0x02}, + {value: 0x0095, lo: 0xbc, hi: 0xbc}, + {value: 0x006d, lo: 0xbd, hi: 0xbd}, + // Block 0x4c, offset 0x1e0 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xaf, hi: 0xb1}, + // Block 0x4d, offset 0x1e2 + {value: 0x0000, lo: 0x02}, + {value: 0x047f, lo: 0xaf, hi: 0xaf}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x4e, offset 0x1e5 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xa0, hi: 0xbf}, + // Block 0x4f, offset 0x1e7 + {value: 0x0000, lo: 0x01}, + {value: 0x0dc3, lo: 0x9f, hi: 0x9f}, + // Block 0x50, offset 0x1e9 + {value: 0x0000, lo: 0x01}, + {value: 0x162f, lo: 0xb3, hi: 0xb3}, + // Block 0x51, offset 0x1eb + {value: 0x0004, lo: 0x0b}, + {value: 0x1597, lo: 0x80, hi: 0x82}, + {value: 0x15af, lo: 0x83, hi: 0x83}, + {value: 0x15c7, lo: 0x84, hi: 0x85}, + {value: 0x15d7, lo: 0x86, hi: 0x89}, + {value: 0x15eb, lo: 0x8a, hi: 0x8c}, + {value: 0x15ff, lo: 0x8d, hi: 0x8d}, + {value: 0x1607, lo: 0x8e, hi: 0x8e}, + {value: 0x160f, lo: 0x8f, hi: 0x90}, + {value: 0x161b, lo: 0x91, hi: 0x93}, + {value: 0x162b, lo: 0x94, hi: 0x94}, + {value: 0x1633, lo: 0x95, hi: 0x95}, + // Block 0x52, offset 0x1f7 + {value: 0x0004, lo: 0x09}, + {value: 0x0001, lo: 0x80, hi: 0x80}, + {value: 0x812c, lo: 0xaa, hi: 0xaa}, + {value: 0x8131, lo: 0xab, hi: 0xab}, + {value: 0x8133, lo: 0xac, hi: 0xac}, + {value: 0x812e, lo: 0xad, hi: 0xad}, + {value: 0x812f, lo: 0xae, hi: 0xae}, + {value: 0x812f, lo: 0xaf, hi: 0xaf}, + {value: 0x04b3, lo: 0xb6, hi: 0xb6}, + {value: 0x0887, lo: 0xb8, hi: 0xba}, + // Block 0x53, offset 0x201 + {value: 0x0005, lo: 0x09}, + {value: 0x0313, lo: 0xb1, hi: 0xb1}, + {value: 0x0317, lo: 0xb2, hi: 0xb2}, + {value: 0x4345, lo: 0xb3, hi: 0xb3}, + {value: 0x031b, lo: 0xb4, hi: 0xb4}, + {value: 0x434a, lo: 0xb5, hi: 0xb6}, + {value: 0x031f, lo: 0xb7, hi: 0xb7}, + {value: 0x0323, lo: 0xb8, hi: 0xb8}, + {value: 0x0327, lo: 0xb9, hi: 0xb9}, + {value: 0x4354, lo: 0xba, hi: 0xbf}, + // Block 0x54, offset 0x20b + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0xaf, hi: 0xaf}, + {value: 0x8132, lo: 0xb4, hi: 0xbd}, + // Block 0x55, offset 0x20e + {value: 0x0000, lo: 0x03}, + {value: 0x020f, lo: 0x9c, hi: 0x9c}, + {value: 0x0212, lo: 0x9d, hi: 0x9d}, + {value: 0x8132, lo: 0x9e, hi: 0x9f}, + // Block 0x56, offset 0x212 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb0, hi: 0xb1}, + // Block 0x57, offset 0x214 + {value: 0x0000, lo: 0x01}, + {value: 0x163b, lo: 0xb0, hi: 0xb0}, + // Block 0x58, offset 0x216 + {value: 0x000c, lo: 0x01}, + {value: 0x00d7, lo: 0xb8, hi: 0xb9}, + // Block 0x59, offset 0x218 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x86, hi: 0x86}, + // Block 0x5a, offset 0x21a + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0xa0, hi: 0xb1}, + // Block 0x5b, offset 0x21d + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xab, hi: 0xad}, + // Block 0x5c, offset 0x21f + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x93, hi: 0x93}, + // Block 0x5d, offset 0x221 + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xb3, hi: 0xb3}, + // Block 0x5e, offset 0x223 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x80, hi: 0x80}, + // Block 0x5f, offset 0x225 + {value: 0x0000, lo: 0x05}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + {value: 0x8132, lo: 0xb2, hi: 0xb3}, + {value: 0x812d, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb7, hi: 0xb8}, + {value: 0x8132, lo: 0xbe, hi: 0xbf}, + // Block 0x60, offset 0x22b + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x81, hi: 0x81}, + {value: 0x8104, lo: 0xb6, hi: 0xb6}, + // Block 0x61, offset 0x22e + {value: 0x0008, lo: 0x03}, + {value: 0x1637, lo: 0x9c, hi: 0x9d}, + {value: 0x0125, lo: 0x9e, hi: 0x9e}, + {value: 0x1643, lo: 0x9f, hi: 0x9f}, + // Block 0x62, offset 0x232 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xad, hi: 0xad}, + // Block 0x63, offset 0x234 + {value: 0x0000, lo: 0x06}, + {value: 0xe500, lo: 0x80, hi: 0x80}, + {value: 0xc600, lo: 0x81, hi: 0x9b}, + {value: 0xe500, lo: 0x9c, hi: 0x9c}, + {value: 0xc600, lo: 0x9d, hi: 0xb7}, + {value: 0xe500, lo: 0xb8, hi: 0xb8}, + {value: 0xc600, lo: 0xb9, hi: 0xbf}, + // Block 0x64, offset 0x23b + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x93}, + {value: 0xe500, lo: 0x94, hi: 0x94}, + {value: 0xc600, lo: 0x95, hi: 0xaf}, + {value: 0xe500, lo: 0xb0, hi: 0xb0}, + {value: 0xc600, lo: 0xb1, hi: 0xbf}, + // Block 0x65, offset 0x241 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x8b}, + {value: 0xe500, lo: 0x8c, hi: 0x8c}, + {value: 0xc600, lo: 0x8d, hi: 0xa7}, + {value: 0xe500, lo: 0xa8, hi: 0xa8}, + {value: 0xc600, lo: 0xa9, hi: 0xbf}, + // Block 0x66, offset 0x247 + {value: 0x0000, lo: 0x07}, + {value: 0xc600, lo: 0x80, hi: 0x83}, + {value: 0xe500, lo: 0x84, hi: 0x84}, + {value: 0xc600, lo: 0x85, hi: 0x9f}, + {value: 0xe500, lo: 0xa0, hi: 0xa0}, + {value: 0xc600, lo: 0xa1, hi: 0xbb}, + {value: 0xe500, lo: 0xbc, hi: 0xbc}, + {value: 0xc600, lo: 0xbd, hi: 0xbf}, + // Block 0x67, offset 0x24f + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x97}, + {value: 0xe500, lo: 0x98, hi: 0x98}, + {value: 0xc600, lo: 0x99, hi: 0xb3}, + {value: 0xe500, lo: 0xb4, hi: 0xb4}, + {value: 0xc600, lo: 0xb5, hi: 0xbf}, + // Block 0x68, offset 0x255 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x8f}, + {value: 0xe500, lo: 0x90, hi: 0x90}, + {value: 0xc600, lo: 0x91, hi: 0xab}, + {value: 0xe500, lo: 0xac, hi: 0xac}, + {value: 0xc600, lo: 0xad, hi: 0xbf}, + // Block 0x69, offset 0x25b + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x87}, + {value: 0xe500, lo: 0x88, hi: 0x88}, + {value: 0xc600, lo: 0x89, hi: 0xa3}, + {value: 0xe500, lo: 0xa4, hi: 0xa4}, + {value: 0xc600, lo: 0xa5, hi: 0xbf}, + // Block 0x6a, offset 0x261 + {value: 0x0000, lo: 0x03}, + {value: 0xc600, lo: 0x80, hi: 0x87}, + {value: 0xe500, lo: 0x88, hi: 0x88}, + {value: 0xc600, lo: 0x89, hi: 0xa3}, + // Block 0x6b, offset 0x265 + {value: 0x0002, lo: 0x01}, + {value: 0x0003, lo: 0x81, hi: 0xbf}, + // Block 0x6c, offset 0x267 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + // Block 0x6d, offset 0x269 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xa0, hi: 0xa0}, + // Block 0x6e, offset 0x26b + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb6, hi: 0xba}, + // Block 0x6f, offset 0x26d + {value: 0x002c, lo: 0x05}, + {value: 0x812d, lo: 0x8d, hi: 0x8d}, + {value: 0x8132, lo: 0x8f, hi: 0x8f}, + {value: 0x8132, lo: 0xb8, hi: 0xb8}, + {value: 0x8101, lo: 0xb9, hi: 0xba}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x70, offset 0x273 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0xa5, hi: 0xa5}, + {value: 0x812d, lo: 0xa6, hi: 0xa6}, + // Block 0x71, offset 0x276 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x86, hi: 0x86}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x72, offset 0x279 + {value: 0x17fe, lo: 0x07}, + {value: 0xa000, lo: 0x99, hi: 0x99}, + {value: 0x4238, lo: 0x9a, hi: 0x9a}, + {value: 0xa000, lo: 0x9b, hi: 0x9b}, + {value: 0x4242, lo: 0x9c, hi: 0x9c}, + {value: 0xa000, lo: 0xa5, hi: 0xa5}, + {value: 0x424c, lo: 0xab, hi: 0xab}, + {value: 0x8104, lo: 0xb9, hi: 0xba}, + // Block 0x73, offset 0x281 + {value: 0x0000, lo: 0x06}, + {value: 0x8132, lo: 0x80, hi: 0x82}, + {value: 0x9900, lo: 0xa7, hi: 0xa7}, + {value: 0x2d7e, lo: 0xae, hi: 0xae}, + {value: 0x2d88, lo: 0xaf, hi: 0xaf}, + {value: 0xa000, lo: 0xb1, hi: 0xb2}, + {value: 0x8104, lo: 0xb3, hi: 0xb4}, + // Block 0x74, offset 0x288 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x80, hi: 0x80}, + {value: 0x8102, lo: 0x8a, hi: 0x8a}, + // Block 0x75, offset 0x28b + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xb5, hi: 0xb5}, + {value: 0x8102, lo: 0xb6, hi: 0xb6}, + // Block 0x76, offset 0x28e + {value: 0x0002, lo: 0x01}, + {value: 0x8102, lo: 0xa9, hi: 0xaa}, + // Block 0x77, offset 0x290 + {value: 0x0000, lo: 0x07}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2d92, lo: 0x8b, hi: 0x8b}, + {value: 0x2d9c, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + {value: 0x8132, lo: 0xa6, hi: 0xac}, + {value: 0x8132, lo: 0xb0, hi: 0xb4}, + // Block 0x78, offset 0x298 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x82, hi: 0x82}, + {value: 0x8102, lo: 0x86, hi: 0x86}, + // Block 0x79, offset 0x29b + {value: 0x6b5a, lo: 0x06}, + {value: 0x9900, lo: 0xb0, hi: 0xb0}, + {value: 0xa000, lo: 0xb9, hi: 0xb9}, + {value: 0x9900, lo: 0xba, hi: 0xba}, + {value: 0x2db0, lo: 0xbb, hi: 0xbb}, + {value: 0x2da6, lo: 0xbc, hi: 0xbd}, + {value: 0x2dba, lo: 0xbe, hi: 0xbe}, + // Block 0x7a, offset 0x2a2 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x82, hi: 0x82}, + {value: 0x8102, lo: 0x83, hi: 0x83}, + // Block 0x7b, offset 0x2a5 + {value: 0x0000, lo: 0x05}, + {value: 0x9900, lo: 0xaf, hi: 0xaf}, + {value: 0xa000, lo: 0xb8, hi: 0xb9}, + {value: 0x2dc4, lo: 0xba, hi: 0xba}, + {value: 0x2dce, lo: 0xbb, hi: 0xbb}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x7c, offset 0x2ab + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0x80, hi: 0x80}, + // Block 0x7d, offset 0x2ad + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x7e, offset 0x2af + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xb6, hi: 0xb6}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + // Block 0x7f, offset 0x2b2 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xab, hi: 0xab}, + // Block 0x80, offset 0x2b4 + {value: 0x0000, lo: 0x01}, + {value: 0x8101, lo: 0xb0, hi: 0xb4}, + // Block 0x81, offset 0x2b6 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb0, hi: 0xb6}, + // Block 0x82, offset 0x2b8 + {value: 0x0000, lo: 0x01}, + {value: 0x8101, lo: 0x9e, hi: 0x9e}, + // Block 0x83, offset 0x2ba + {value: 0x0000, lo: 0x0c}, + {value: 0x4662, lo: 0x9e, hi: 0x9e}, + {value: 0x466c, lo: 0x9f, hi: 0x9f}, + {value: 0x46a0, lo: 0xa0, hi: 0xa0}, + {value: 0x46ae, lo: 0xa1, hi: 0xa1}, + {value: 0x46bc, lo: 0xa2, hi: 0xa2}, + {value: 0x46ca, lo: 0xa3, hi: 0xa3}, + {value: 0x46d8, lo: 0xa4, hi: 0xa4}, + {value: 0x812b, lo: 0xa5, hi: 0xa6}, + {value: 0x8101, lo: 0xa7, hi: 0xa9}, + {value: 0x8130, lo: 0xad, hi: 0xad}, + {value: 0x812b, lo: 0xae, hi: 0xb2}, + {value: 0x812d, lo: 0xbb, hi: 0xbf}, + // Block 0x84, offset 0x2c7 + {value: 0x0000, lo: 0x09}, + {value: 0x812d, lo: 0x80, hi: 0x82}, + {value: 0x8132, lo: 0x85, hi: 0x89}, + {value: 0x812d, lo: 0x8a, hi: 0x8b}, + {value: 0x8132, lo: 0xaa, hi: 0xad}, + {value: 0x4676, lo: 0xbb, hi: 0xbb}, + {value: 0x4680, lo: 0xbc, hi: 0xbc}, + {value: 0x46e6, lo: 0xbd, hi: 0xbd}, + {value: 0x4702, lo: 0xbe, hi: 0xbe}, + {value: 0x46f4, lo: 0xbf, hi: 0xbf}, + // Block 0x85, offset 0x2d1 + {value: 0x0000, lo: 0x01}, + {value: 0x4710, lo: 0x80, hi: 0x80}, + // Block 0x86, offset 0x2d3 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x82, hi: 0x84}, + // Block 0x87, offset 0x2d5 + {value: 0x0002, lo: 0x03}, + {value: 0x0043, lo: 0x80, hi: 0x99}, + {value: 0x0083, lo: 0x9a, hi: 0xb3}, + {value: 0x0043, lo: 0xb4, hi: 0xbf}, + // Block 0x88, offset 0x2d9 + {value: 0x0002, lo: 0x04}, + {value: 0x005b, lo: 0x80, hi: 0x8d}, + {value: 0x0083, lo: 0x8e, hi: 0x94}, + {value: 0x0093, lo: 0x96, hi: 0xa7}, + {value: 0x0043, lo: 0xa8, hi: 0xbf}, + // Block 0x89, offset 0x2de + {value: 0x0002, lo: 0x0b}, + {value: 0x0073, lo: 0x80, hi: 0x81}, + {value: 0x0083, lo: 0x82, hi: 0x9b}, + {value: 0x0043, lo: 0x9c, hi: 0x9c}, + {value: 0x0047, lo: 0x9e, hi: 0x9f}, + {value: 0x004f, lo: 0xa2, hi: 0xa2}, + {value: 0x0055, lo: 0xa5, hi: 0xa6}, + {value: 0x005d, lo: 0xa9, hi: 0xac}, + {value: 0x0067, lo: 0xae, hi: 0xb5}, + {value: 0x0083, lo: 0xb6, hi: 0xb9}, + {value: 0x008d, lo: 0xbb, hi: 0xbb}, + {value: 0x0091, lo: 0xbd, hi: 0xbf}, + // Block 0x8a, offset 0x2ea + {value: 0x0002, lo: 0x04}, + {value: 0x0097, lo: 0x80, hi: 0x83}, + {value: 0x00a1, lo: 0x85, hi: 0x8f}, + {value: 0x0043, lo: 0x90, hi: 0xa9}, + {value: 0x0083, lo: 0xaa, hi: 0xbf}, + // Block 0x8b, offset 0x2ef + {value: 0x0002, lo: 0x08}, + {value: 0x00af, lo: 0x80, hi: 0x83}, + {value: 0x0043, lo: 0x84, hi: 0x85}, + {value: 0x0049, lo: 0x87, hi: 0x8a}, + {value: 0x0055, lo: 0x8d, hi: 0x94}, + {value: 0x0067, lo: 0x96, hi: 0x9c}, + {value: 0x0083, lo: 0x9e, hi: 0xb7}, + {value: 0x0043, lo: 0xb8, hi: 0xb9}, + {value: 0x0049, lo: 0xbb, hi: 0xbe}, + // Block 0x8c, offset 0x2f8 + {value: 0x0002, lo: 0x05}, + {value: 0x0053, lo: 0x80, hi: 0x84}, + {value: 0x005f, lo: 0x86, hi: 0x86}, + {value: 0x0067, lo: 0x8a, hi: 0x90}, + {value: 0x0083, lo: 0x92, hi: 0xab}, + {value: 0x0043, lo: 0xac, hi: 0xbf}, + // Block 0x8d, offset 0x2fe + {value: 0x0002, lo: 0x04}, + {value: 0x006b, lo: 0x80, hi: 0x85}, + {value: 0x0083, lo: 0x86, hi: 0x9f}, + {value: 0x0043, lo: 0xa0, hi: 0xb9}, + {value: 0x0083, lo: 0xba, hi: 0xbf}, + // Block 0x8e, offset 0x303 + {value: 0x0002, lo: 0x03}, + {value: 0x008f, lo: 0x80, hi: 0x93}, + {value: 0x0043, lo: 0x94, hi: 0xad}, + {value: 0x0083, lo: 0xae, hi: 0xbf}, + // Block 0x8f, offset 0x307 + {value: 0x0002, lo: 0x04}, + {value: 0x00a7, lo: 0x80, hi: 0x87}, + {value: 0x0043, lo: 0x88, hi: 0xa1}, + {value: 0x0083, lo: 0xa2, hi: 0xbb}, + {value: 0x0043, lo: 0xbc, hi: 0xbf}, + // Block 0x90, offset 0x30c + {value: 0x0002, lo: 0x03}, + {value: 0x004b, lo: 0x80, hi: 0x95}, + {value: 0x0083, lo: 0x96, hi: 0xaf}, + {value: 0x0043, lo: 0xb0, hi: 0xbf}, + // Block 0x91, offset 0x310 + {value: 0x0003, lo: 0x0f}, + {value: 0x01b8, lo: 0x80, hi: 0x80}, + {value: 0x045f, lo: 0x81, hi: 0x81}, + {value: 0x01bb, lo: 0x82, hi: 0x9a}, + {value: 0x045b, lo: 0x9b, hi: 0x9b}, + {value: 0x01c7, lo: 0x9c, hi: 0x9c}, + {value: 0x01d0, lo: 0x9d, hi: 0x9d}, + {value: 0x01d6, lo: 0x9e, hi: 0x9e}, + {value: 0x01fa, lo: 0x9f, hi: 0x9f}, + {value: 0x01eb, lo: 0xa0, hi: 0xa0}, + {value: 0x01e8, lo: 0xa1, hi: 0xa1}, + {value: 0x0173, lo: 0xa2, hi: 0xb2}, + {value: 0x0188, lo: 0xb3, hi: 0xb3}, + {value: 0x01a6, lo: 0xb4, hi: 0xba}, + {value: 0x045f, lo: 0xbb, hi: 0xbb}, + {value: 0x01bb, lo: 0xbc, hi: 0xbf}, + // Block 0x92, offset 0x320 + {value: 0x0003, lo: 0x0d}, + {value: 0x01c7, lo: 0x80, hi: 0x94}, + {value: 0x045b, lo: 0x95, hi: 0x95}, + {value: 0x01c7, lo: 0x96, hi: 0x96}, + {value: 0x01d0, lo: 0x97, hi: 0x97}, + {value: 0x01d6, lo: 0x98, hi: 0x98}, + {value: 0x01fa, lo: 0x99, hi: 0x99}, + {value: 0x01eb, lo: 0x9a, hi: 0x9a}, + {value: 0x01e8, lo: 0x9b, hi: 0x9b}, + {value: 0x0173, lo: 0x9c, hi: 0xac}, + {value: 0x0188, lo: 0xad, hi: 0xad}, + {value: 0x01a6, lo: 0xae, hi: 0xb4}, + {value: 0x045f, lo: 0xb5, hi: 0xb5}, + {value: 0x01bb, lo: 0xb6, hi: 0xbf}, + // Block 0x93, offset 0x32e + {value: 0x0003, lo: 0x0d}, + {value: 0x01d9, lo: 0x80, hi: 0x8e}, + {value: 0x045b, lo: 0x8f, hi: 0x8f}, + {value: 0x01c7, lo: 0x90, hi: 0x90}, + {value: 0x01d0, lo: 0x91, hi: 0x91}, + {value: 0x01d6, lo: 0x92, hi: 0x92}, + {value: 0x01fa, lo: 0x93, hi: 0x93}, + {value: 0x01eb, lo: 0x94, hi: 0x94}, + {value: 0x01e8, lo: 0x95, hi: 0x95}, + {value: 0x0173, lo: 0x96, hi: 0xa6}, + {value: 0x0188, lo: 0xa7, hi: 0xa7}, + {value: 0x01a6, lo: 0xa8, hi: 0xae}, + {value: 0x045f, lo: 0xaf, hi: 0xaf}, + {value: 0x01bb, lo: 0xb0, hi: 0xbf}, + // Block 0x94, offset 0x33c + {value: 0x0003, lo: 0x0d}, + {value: 0x01eb, lo: 0x80, hi: 0x88}, + {value: 0x045b, lo: 0x89, hi: 0x89}, + {value: 0x01c7, lo: 0x8a, hi: 0x8a}, + {value: 0x01d0, lo: 0x8b, hi: 0x8b}, + {value: 0x01d6, lo: 0x8c, hi: 0x8c}, + {value: 0x01fa, lo: 0x8d, hi: 0x8d}, + {value: 0x01eb, lo: 0x8e, hi: 0x8e}, + {value: 0x01e8, lo: 0x8f, hi: 0x8f}, + {value: 0x0173, lo: 0x90, hi: 0xa0}, + {value: 0x0188, lo: 0xa1, hi: 0xa1}, + {value: 0x01a6, lo: 0xa2, hi: 0xa8}, + {value: 0x045f, lo: 0xa9, hi: 0xa9}, + {value: 0x01bb, lo: 0xaa, hi: 0xbf}, + // Block 0x95, offset 0x34a + {value: 0x0000, lo: 0x05}, + {value: 0x8132, lo: 0x80, hi: 0x86}, + {value: 0x8132, lo: 0x88, hi: 0x98}, + {value: 0x8132, lo: 0x9b, hi: 0xa1}, + {value: 0x8132, lo: 0xa3, hi: 0xa4}, + {value: 0x8132, lo: 0xa6, hi: 0xaa}, + // Block 0x96, offset 0x350 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x90, hi: 0x96}, + // Block 0x97, offset 0x352 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x84, hi: 0x89}, + {value: 0x8102, lo: 0x8a, hi: 0x8a}, + // Block 0x98, offset 0x355 + {value: 0x0002, lo: 0x09}, + {value: 0x0063, lo: 0x80, hi: 0x89}, + {value: 0x1951, lo: 0x8a, hi: 0x8a}, + {value: 0x1981, lo: 0x8b, hi: 0x8b}, + {value: 0x199c, lo: 0x8c, hi: 0x8c}, + {value: 0x19a2, lo: 0x8d, hi: 0x8d}, + {value: 0x1bc0, lo: 0x8e, hi: 0x8e}, + {value: 0x19ae, lo: 0x8f, hi: 0x8f}, + {value: 0x197b, lo: 0xaa, hi: 0xaa}, + {value: 0x197e, lo: 0xab, hi: 0xab}, + // Block 0x99, offset 0x35f + {value: 0x0000, lo: 0x01}, + {value: 0x193f, lo: 0x90, hi: 0x90}, + // Block 0x9a, offset 0x361 + {value: 0x0028, lo: 0x09}, + {value: 0x2862, lo: 0x80, hi: 0x80}, + {value: 0x2826, lo: 0x81, hi: 0x81}, + {value: 0x2830, lo: 0x82, hi: 0x82}, + {value: 0x2844, lo: 0x83, hi: 0x84}, + {value: 0x284e, lo: 0x85, hi: 0x86}, + {value: 0x283a, lo: 0x87, hi: 0x87}, + {value: 0x2858, lo: 0x88, hi: 0x88}, + {value: 0x0b6f, lo: 0x90, hi: 0x90}, + {value: 0x08e7, lo: 0x91, hi: 0x91}, +} + +// recompMap: 7520 bytes (entries only) +var recompMap = map[uint32]rune{ + 0x00410300: 0x00C0, + 0x00410301: 0x00C1, + 0x00410302: 0x00C2, + 0x00410303: 0x00C3, + 0x00410308: 0x00C4, + 0x0041030A: 0x00C5, + 0x00430327: 0x00C7, + 0x00450300: 0x00C8, + 0x00450301: 0x00C9, + 0x00450302: 0x00CA, + 0x00450308: 0x00CB, + 0x00490300: 0x00CC, + 0x00490301: 0x00CD, + 0x00490302: 0x00CE, + 0x00490308: 0x00CF, + 0x004E0303: 0x00D1, + 0x004F0300: 0x00D2, + 0x004F0301: 0x00D3, + 0x004F0302: 0x00D4, + 0x004F0303: 0x00D5, + 0x004F0308: 0x00D6, + 0x00550300: 0x00D9, + 0x00550301: 0x00DA, + 0x00550302: 0x00DB, + 0x00550308: 0x00DC, + 0x00590301: 0x00DD, + 0x00610300: 0x00E0, + 0x00610301: 0x00E1, + 0x00610302: 0x00E2, + 0x00610303: 0x00E3, + 0x00610308: 0x00E4, + 0x0061030A: 0x00E5, + 0x00630327: 0x00E7, + 0x00650300: 0x00E8, + 0x00650301: 0x00E9, + 0x00650302: 0x00EA, + 0x00650308: 0x00EB, + 0x00690300: 0x00EC, + 0x00690301: 0x00ED, + 0x00690302: 0x00EE, + 0x00690308: 0x00EF, + 0x006E0303: 0x00F1, + 0x006F0300: 0x00F2, + 0x006F0301: 0x00F3, + 0x006F0302: 0x00F4, + 0x006F0303: 0x00F5, + 0x006F0308: 0x00F6, + 0x00750300: 0x00F9, + 0x00750301: 0x00FA, + 0x00750302: 0x00FB, + 0x00750308: 0x00FC, + 0x00790301: 0x00FD, + 0x00790308: 0x00FF, + 0x00410304: 0x0100, + 0x00610304: 0x0101, + 0x00410306: 0x0102, + 0x00610306: 0x0103, + 0x00410328: 0x0104, + 0x00610328: 0x0105, + 0x00430301: 0x0106, + 0x00630301: 0x0107, + 0x00430302: 0x0108, + 0x00630302: 0x0109, + 0x00430307: 0x010A, + 0x00630307: 0x010B, + 0x0043030C: 0x010C, + 0x0063030C: 0x010D, + 0x0044030C: 0x010E, + 0x0064030C: 0x010F, + 0x00450304: 0x0112, + 0x00650304: 0x0113, + 0x00450306: 0x0114, + 0x00650306: 0x0115, + 0x00450307: 0x0116, + 0x00650307: 0x0117, + 0x00450328: 0x0118, + 0x00650328: 0x0119, + 0x0045030C: 0x011A, + 0x0065030C: 0x011B, + 0x00470302: 0x011C, + 0x00670302: 0x011D, + 0x00470306: 0x011E, + 0x00670306: 0x011F, + 0x00470307: 0x0120, + 0x00670307: 0x0121, + 0x00470327: 0x0122, + 0x00670327: 0x0123, + 0x00480302: 0x0124, + 0x00680302: 0x0125, + 0x00490303: 0x0128, + 0x00690303: 0x0129, + 0x00490304: 0x012A, + 0x00690304: 0x012B, + 0x00490306: 0x012C, + 0x00690306: 0x012D, + 0x00490328: 0x012E, + 0x00690328: 0x012F, + 0x00490307: 0x0130, + 0x004A0302: 0x0134, + 0x006A0302: 0x0135, + 0x004B0327: 0x0136, + 0x006B0327: 0x0137, + 0x004C0301: 0x0139, + 0x006C0301: 0x013A, + 0x004C0327: 0x013B, + 0x006C0327: 0x013C, + 0x004C030C: 0x013D, + 0x006C030C: 0x013E, + 0x004E0301: 0x0143, + 0x006E0301: 0x0144, + 0x004E0327: 0x0145, + 0x006E0327: 0x0146, + 0x004E030C: 0x0147, + 0x006E030C: 0x0148, + 0x004F0304: 0x014C, + 0x006F0304: 0x014D, + 0x004F0306: 0x014E, + 0x006F0306: 0x014F, + 0x004F030B: 0x0150, + 0x006F030B: 0x0151, + 0x00520301: 0x0154, + 0x00720301: 0x0155, + 0x00520327: 0x0156, + 0x00720327: 0x0157, + 0x0052030C: 0x0158, + 0x0072030C: 0x0159, + 0x00530301: 0x015A, + 0x00730301: 0x015B, + 0x00530302: 0x015C, + 0x00730302: 0x015D, + 0x00530327: 0x015E, + 0x00730327: 0x015F, + 0x0053030C: 0x0160, + 0x0073030C: 0x0161, + 0x00540327: 0x0162, + 0x00740327: 0x0163, + 0x0054030C: 0x0164, + 0x0074030C: 0x0165, + 0x00550303: 0x0168, + 0x00750303: 0x0169, + 0x00550304: 0x016A, + 0x00750304: 0x016B, + 0x00550306: 0x016C, + 0x00750306: 0x016D, + 0x0055030A: 0x016E, + 0x0075030A: 0x016F, + 0x0055030B: 0x0170, + 0x0075030B: 0x0171, + 0x00550328: 0x0172, + 0x00750328: 0x0173, + 0x00570302: 0x0174, + 0x00770302: 0x0175, + 0x00590302: 0x0176, + 0x00790302: 0x0177, + 0x00590308: 0x0178, + 0x005A0301: 0x0179, + 0x007A0301: 0x017A, + 0x005A0307: 0x017B, + 0x007A0307: 0x017C, + 0x005A030C: 0x017D, + 0x007A030C: 0x017E, + 0x004F031B: 0x01A0, + 0x006F031B: 0x01A1, + 0x0055031B: 0x01AF, + 0x0075031B: 0x01B0, + 0x0041030C: 0x01CD, + 0x0061030C: 0x01CE, + 0x0049030C: 0x01CF, + 0x0069030C: 0x01D0, + 0x004F030C: 0x01D1, + 0x006F030C: 0x01D2, + 0x0055030C: 0x01D3, + 0x0075030C: 0x01D4, + 0x00DC0304: 0x01D5, + 0x00FC0304: 0x01D6, + 0x00DC0301: 0x01D7, + 0x00FC0301: 0x01D8, + 0x00DC030C: 0x01D9, + 0x00FC030C: 0x01DA, + 0x00DC0300: 0x01DB, + 0x00FC0300: 0x01DC, + 0x00C40304: 0x01DE, + 0x00E40304: 0x01DF, + 0x02260304: 0x01E0, + 0x02270304: 0x01E1, + 0x00C60304: 0x01E2, + 0x00E60304: 0x01E3, + 0x0047030C: 0x01E6, + 0x0067030C: 0x01E7, + 0x004B030C: 0x01E8, + 0x006B030C: 0x01E9, + 0x004F0328: 0x01EA, + 0x006F0328: 0x01EB, + 0x01EA0304: 0x01EC, + 0x01EB0304: 0x01ED, + 0x01B7030C: 0x01EE, + 0x0292030C: 0x01EF, + 0x006A030C: 0x01F0, + 0x00470301: 0x01F4, + 0x00670301: 0x01F5, + 0x004E0300: 0x01F8, + 0x006E0300: 0x01F9, + 0x00C50301: 0x01FA, + 0x00E50301: 0x01FB, + 0x00C60301: 0x01FC, + 0x00E60301: 0x01FD, + 0x00D80301: 0x01FE, + 0x00F80301: 0x01FF, + 0x0041030F: 0x0200, + 0x0061030F: 0x0201, + 0x00410311: 0x0202, + 0x00610311: 0x0203, + 0x0045030F: 0x0204, + 0x0065030F: 0x0205, + 0x00450311: 0x0206, + 0x00650311: 0x0207, + 0x0049030F: 0x0208, + 0x0069030F: 0x0209, + 0x00490311: 0x020A, + 0x00690311: 0x020B, + 0x004F030F: 0x020C, + 0x006F030F: 0x020D, + 0x004F0311: 0x020E, + 0x006F0311: 0x020F, + 0x0052030F: 0x0210, + 0x0072030F: 0x0211, + 0x00520311: 0x0212, + 0x00720311: 0x0213, + 0x0055030F: 0x0214, + 0x0075030F: 0x0215, + 0x00550311: 0x0216, + 0x00750311: 0x0217, + 0x00530326: 0x0218, + 0x00730326: 0x0219, + 0x00540326: 0x021A, + 0x00740326: 0x021B, + 0x0048030C: 0x021E, + 0x0068030C: 0x021F, + 0x00410307: 0x0226, + 0x00610307: 0x0227, + 0x00450327: 0x0228, + 0x00650327: 0x0229, + 0x00D60304: 0x022A, + 0x00F60304: 0x022B, + 0x00D50304: 0x022C, + 0x00F50304: 0x022D, + 0x004F0307: 0x022E, + 0x006F0307: 0x022F, + 0x022E0304: 0x0230, + 0x022F0304: 0x0231, + 0x00590304: 0x0232, + 0x00790304: 0x0233, + 0x00A80301: 0x0385, + 0x03910301: 0x0386, + 0x03950301: 0x0388, + 0x03970301: 0x0389, + 0x03990301: 0x038A, + 0x039F0301: 0x038C, + 0x03A50301: 0x038E, + 0x03A90301: 0x038F, + 0x03CA0301: 0x0390, + 0x03990308: 0x03AA, + 0x03A50308: 0x03AB, + 0x03B10301: 0x03AC, + 0x03B50301: 0x03AD, + 0x03B70301: 0x03AE, + 0x03B90301: 0x03AF, + 0x03CB0301: 0x03B0, + 0x03B90308: 0x03CA, + 0x03C50308: 0x03CB, + 0x03BF0301: 0x03CC, + 0x03C50301: 0x03CD, + 0x03C90301: 0x03CE, + 0x03D20301: 0x03D3, + 0x03D20308: 0x03D4, + 0x04150300: 0x0400, + 0x04150308: 0x0401, + 0x04130301: 0x0403, + 0x04060308: 0x0407, + 0x041A0301: 0x040C, + 0x04180300: 0x040D, + 0x04230306: 0x040E, + 0x04180306: 0x0419, + 0x04380306: 0x0439, + 0x04350300: 0x0450, + 0x04350308: 0x0451, + 0x04330301: 0x0453, + 0x04560308: 0x0457, + 0x043A0301: 0x045C, + 0x04380300: 0x045D, + 0x04430306: 0x045E, + 0x0474030F: 0x0476, + 0x0475030F: 0x0477, + 0x04160306: 0x04C1, + 0x04360306: 0x04C2, + 0x04100306: 0x04D0, + 0x04300306: 0x04D1, + 0x04100308: 0x04D2, + 0x04300308: 0x04D3, + 0x04150306: 0x04D6, + 0x04350306: 0x04D7, + 0x04D80308: 0x04DA, + 0x04D90308: 0x04DB, + 0x04160308: 0x04DC, + 0x04360308: 0x04DD, + 0x04170308: 0x04DE, + 0x04370308: 0x04DF, + 0x04180304: 0x04E2, + 0x04380304: 0x04E3, + 0x04180308: 0x04E4, + 0x04380308: 0x04E5, + 0x041E0308: 0x04E6, + 0x043E0308: 0x04E7, + 0x04E80308: 0x04EA, + 0x04E90308: 0x04EB, + 0x042D0308: 0x04EC, + 0x044D0308: 0x04ED, + 0x04230304: 0x04EE, + 0x04430304: 0x04EF, + 0x04230308: 0x04F0, + 0x04430308: 0x04F1, + 0x0423030B: 0x04F2, + 0x0443030B: 0x04F3, + 0x04270308: 0x04F4, + 0x04470308: 0x04F5, + 0x042B0308: 0x04F8, + 0x044B0308: 0x04F9, + 0x06270653: 0x0622, + 0x06270654: 0x0623, + 0x06480654: 0x0624, + 0x06270655: 0x0625, + 0x064A0654: 0x0626, + 0x06D50654: 0x06C0, + 0x06C10654: 0x06C2, + 0x06D20654: 0x06D3, + 0x0928093C: 0x0929, + 0x0930093C: 0x0931, + 0x0933093C: 0x0934, + 0x09C709BE: 0x09CB, + 0x09C709D7: 0x09CC, + 0x0B470B56: 0x0B48, + 0x0B470B3E: 0x0B4B, + 0x0B470B57: 0x0B4C, + 0x0B920BD7: 0x0B94, + 0x0BC60BBE: 0x0BCA, + 0x0BC70BBE: 0x0BCB, + 0x0BC60BD7: 0x0BCC, + 0x0C460C56: 0x0C48, + 0x0CBF0CD5: 0x0CC0, + 0x0CC60CD5: 0x0CC7, + 0x0CC60CD6: 0x0CC8, + 0x0CC60CC2: 0x0CCA, + 0x0CCA0CD5: 0x0CCB, + 0x0D460D3E: 0x0D4A, + 0x0D470D3E: 0x0D4B, + 0x0D460D57: 0x0D4C, + 0x0DD90DCA: 0x0DDA, + 0x0DD90DCF: 0x0DDC, + 0x0DDC0DCA: 0x0DDD, + 0x0DD90DDF: 0x0DDE, + 0x1025102E: 0x1026, + 0x1B051B35: 0x1B06, + 0x1B071B35: 0x1B08, + 0x1B091B35: 0x1B0A, + 0x1B0B1B35: 0x1B0C, + 0x1B0D1B35: 0x1B0E, + 0x1B111B35: 0x1B12, + 0x1B3A1B35: 0x1B3B, + 0x1B3C1B35: 0x1B3D, + 0x1B3E1B35: 0x1B40, + 0x1B3F1B35: 0x1B41, + 0x1B421B35: 0x1B43, + 0x00410325: 0x1E00, + 0x00610325: 0x1E01, + 0x00420307: 0x1E02, + 0x00620307: 0x1E03, + 0x00420323: 0x1E04, + 0x00620323: 0x1E05, + 0x00420331: 0x1E06, + 0x00620331: 0x1E07, + 0x00C70301: 0x1E08, + 0x00E70301: 0x1E09, + 0x00440307: 0x1E0A, + 0x00640307: 0x1E0B, + 0x00440323: 0x1E0C, + 0x00640323: 0x1E0D, + 0x00440331: 0x1E0E, + 0x00640331: 0x1E0F, + 0x00440327: 0x1E10, + 0x00640327: 0x1E11, + 0x0044032D: 0x1E12, + 0x0064032D: 0x1E13, + 0x01120300: 0x1E14, + 0x01130300: 0x1E15, + 0x01120301: 0x1E16, + 0x01130301: 0x1E17, + 0x0045032D: 0x1E18, + 0x0065032D: 0x1E19, + 0x00450330: 0x1E1A, + 0x00650330: 0x1E1B, + 0x02280306: 0x1E1C, + 0x02290306: 0x1E1D, + 0x00460307: 0x1E1E, + 0x00660307: 0x1E1F, + 0x00470304: 0x1E20, + 0x00670304: 0x1E21, + 0x00480307: 0x1E22, + 0x00680307: 0x1E23, + 0x00480323: 0x1E24, + 0x00680323: 0x1E25, + 0x00480308: 0x1E26, + 0x00680308: 0x1E27, + 0x00480327: 0x1E28, + 0x00680327: 0x1E29, + 0x0048032E: 0x1E2A, + 0x0068032E: 0x1E2B, + 0x00490330: 0x1E2C, + 0x00690330: 0x1E2D, + 0x00CF0301: 0x1E2E, + 0x00EF0301: 0x1E2F, + 0x004B0301: 0x1E30, + 0x006B0301: 0x1E31, + 0x004B0323: 0x1E32, + 0x006B0323: 0x1E33, + 0x004B0331: 0x1E34, + 0x006B0331: 0x1E35, + 0x004C0323: 0x1E36, + 0x006C0323: 0x1E37, + 0x1E360304: 0x1E38, + 0x1E370304: 0x1E39, + 0x004C0331: 0x1E3A, + 0x006C0331: 0x1E3B, + 0x004C032D: 0x1E3C, + 0x006C032D: 0x1E3D, + 0x004D0301: 0x1E3E, + 0x006D0301: 0x1E3F, + 0x004D0307: 0x1E40, + 0x006D0307: 0x1E41, + 0x004D0323: 0x1E42, + 0x006D0323: 0x1E43, + 0x004E0307: 0x1E44, + 0x006E0307: 0x1E45, + 0x004E0323: 0x1E46, + 0x006E0323: 0x1E47, + 0x004E0331: 0x1E48, + 0x006E0331: 0x1E49, + 0x004E032D: 0x1E4A, + 0x006E032D: 0x1E4B, + 0x00D50301: 0x1E4C, + 0x00F50301: 0x1E4D, + 0x00D50308: 0x1E4E, + 0x00F50308: 0x1E4F, + 0x014C0300: 0x1E50, + 0x014D0300: 0x1E51, + 0x014C0301: 0x1E52, + 0x014D0301: 0x1E53, + 0x00500301: 0x1E54, + 0x00700301: 0x1E55, + 0x00500307: 0x1E56, + 0x00700307: 0x1E57, + 0x00520307: 0x1E58, + 0x00720307: 0x1E59, + 0x00520323: 0x1E5A, + 0x00720323: 0x1E5B, + 0x1E5A0304: 0x1E5C, + 0x1E5B0304: 0x1E5D, + 0x00520331: 0x1E5E, + 0x00720331: 0x1E5F, + 0x00530307: 0x1E60, + 0x00730307: 0x1E61, + 0x00530323: 0x1E62, + 0x00730323: 0x1E63, + 0x015A0307: 0x1E64, + 0x015B0307: 0x1E65, + 0x01600307: 0x1E66, + 0x01610307: 0x1E67, + 0x1E620307: 0x1E68, + 0x1E630307: 0x1E69, + 0x00540307: 0x1E6A, + 0x00740307: 0x1E6B, + 0x00540323: 0x1E6C, + 0x00740323: 0x1E6D, + 0x00540331: 0x1E6E, + 0x00740331: 0x1E6F, + 0x0054032D: 0x1E70, + 0x0074032D: 0x1E71, + 0x00550324: 0x1E72, + 0x00750324: 0x1E73, + 0x00550330: 0x1E74, + 0x00750330: 0x1E75, + 0x0055032D: 0x1E76, + 0x0075032D: 0x1E77, + 0x01680301: 0x1E78, + 0x01690301: 0x1E79, + 0x016A0308: 0x1E7A, + 0x016B0308: 0x1E7B, + 0x00560303: 0x1E7C, + 0x00760303: 0x1E7D, + 0x00560323: 0x1E7E, + 0x00760323: 0x1E7F, + 0x00570300: 0x1E80, + 0x00770300: 0x1E81, + 0x00570301: 0x1E82, + 0x00770301: 0x1E83, + 0x00570308: 0x1E84, + 0x00770308: 0x1E85, + 0x00570307: 0x1E86, + 0x00770307: 0x1E87, + 0x00570323: 0x1E88, + 0x00770323: 0x1E89, + 0x00580307: 0x1E8A, + 0x00780307: 0x1E8B, + 0x00580308: 0x1E8C, + 0x00780308: 0x1E8D, + 0x00590307: 0x1E8E, + 0x00790307: 0x1E8F, + 0x005A0302: 0x1E90, + 0x007A0302: 0x1E91, + 0x005A0323: 0x1E92, + 0x007A0323: 0x1E93, + 0x005A0331: 0x1E94, + 0x007A0331: 0x1E95, + 0x00680331: 0x1E96, + 0x00740308: 0x1E97, + 0x0077030A: 0x1E98, + 0x0079030A: 0x1E99, + 0x017F0307: 0x1E9B, + 0x00410323: 0x1EA0, + 0x00610323: 0x1EA1, + 0x00410309: 0x1EA2, + 0x00610309: 0x1EA3, + 0x00C20301: 0x1EA4, + 0x00E20301: 0x1EA5, + 0x00C20300: 0x1EA6, + 0x00E20300: 0x1EA7, + 0x00C20309: 0x1EA8, + 0x00E20309: 0x1EA9, + 0x00C20303: 0x1EAA, + 0x00E20303: 0x1EAB, + 0x1EA00302: 0x1EAC, + 0x1EA10302: 0x1EAD, + 0x01020301: 0x1EAE, + 0x01030301: 0x1EAF, + 0x01020300: 0x1EB0, + 0x01030300: 0x1EB1, + 0x01020309: 0x1EB2, + 0x01030309: 0x1EB3, + 0x01020303: 0x1EB4, + 0x01030303: 0x1EB5, + 0x1EA00306: 0x1EB6, + 0x1EA10306: 0x1EB7, + 0x00450323: 0x1EB8, + 0x00650323: 0x1EB9, + 0x00450309: 0x1EBA, + 0x00650309: 0x1EBB, + 0x00450303: 0x1EBC, + 0x00650303: 0x1EBD, + 0x00CA0301: 0x1EBE, + 0x00EA0301: 0x1EBF, + 0x00CA0300: 0x1EC0, + 0x00EA0300: 0x1EC1, + 0x00CA0309: 0x1EC2, + 0x00EA0309: 0x1EC3, + 0x00CA0303: 0x1EC4, + 0x00EA0303: 0x1EC5, + 0x1EB80302: 0x1EC6, + 0x1EB90302: 0x1EC7, + 0x00490309: 0x1EC8, + 0x00690309: 0x1EC9, + 0x00490323: 0x1ECA, + 0x00690323: 0x1ECB, + 0x004F0323: 0x1ECC, + 0x006F0323: 0x1ECD, + 0x004F0309: 0x1ECE, + 0x006F0309: 0x1ECF, + 0x00D40301: 0x1ED0, + 0x00F40301: 0x1ED1, + 0x00D40300: 0x1ED2, + 0x00F40300: 0x1ED3, + 0x00D40309: 0x1ED4, + 0x00F40309: 0x1ED5, + 0x00D40303: 0x1ED6, + 0x00F40303: 0x1ED7, + 0x1ECC0302: 0x1ED8, + 0x1ECD0302: 0x1ED9, + 0x01A00301: 0x1EDA, + 0x01A10301: 0x1EDB, + 0x01A00300: 0x1EDC, + 0x01A10300: 0x1EDD, + 0x01A00309: 0x1EDE, + 0x01A10309: 0x1EDF, + 0x01A00303: 0x1EE0, + 0x01A10303: 0x1EE1, + 0x01A00323: 0x1EE2, + 0x01A10323: 0x1EE3, + 0x00550323: 0x1EE4, + 0x00750323: 0x1EE5, + 0x00550309: 0x1EE6, + 0x00750309: 0x1EE7, + 0x01AF0301: 0x1EE8, + 0x01B00301: 0x1EE9, + 0x01AF0300: 0x1EEA, + 0x01B00300: 0x1EEB, + 0x01AF0309: 0x1EEC, + 0x01B00309: 0x1EED, + 0x01AF0303: 0x1EEE, + 0x01B00303: 0x1EEF, + 0x01AF0323: 0x1EF0, + 0x01B00323: 0x1EF1, + 0x00590300: 0x1EF2, + 0x00790300: 0x1EF3, + 0x00590323: 0x1EF4, + 0x00790323: 0x1EF5, + 0x00590309: 0x1EF6, + 0x00790309: 0x1EF7, + 0x00590303: 0x1EF8, + 0x00790303: 0x1EF9, + 0x03B10313: 0x1F00, + 0x03B10314: 0x1F01, + 0x1F000300: 0x1F02, + 0x1F010300: 0x1F03, + 0x1F000301: 0x1F04, + 0x1F010301: 0x1F05, + 0x1F000342: 0x1F06, + 0x1F010342: 0x1F07, + 0x03910313: 0x1F08, + 0x03910314: 0x1F09, + 0x1F080300: 0x1F0A, + 0x1F090300: 0x1F0B, + 0x1F080301: 0x1F0C, + 0x1F090301: 0x1F0D, + 0x1F080342: 0x1F0E, + 0x1F090342: 0x1F0F, + 0x03B50313: 0x1F10, + 0x03B50314: 0x1F11, + 0x1F100300: 0x1F12, + 0x1F110300: 0x1F13, + 0x1F100301: 0x1F14, + 0x1F110301: 0x1F15, + 0x03950313: 0x1F18, + 0x03950314: 0x1F19, + 0x1F180300: 0x1F1A, + 0x1F190300: 0x1F1B, + 0x1F180301: 0x1F1C, + 0x1F190301: 0x1F1D, + 0x03B70313: 0x1F20, + 0x03B70314: 0x1F21, + 0x1F200300: 0x1F22, + 0x1F210300: 0x1F23, + 0x1F200301: 0x1F24, + 0x1F210301: 0x1F25, + 0x1F200342: 0x1F26, + 0x1F210342: 0x1F27, + 0x03970313: 0x1F28, + 0x03970314: 0x1F29, + 0x1F280300: 0x1F2A, + 0x1F290300: 0x1F2B, + 0x1F280301: 0x1F2C, + 0x1F290301: 0x1F2D, + 0x1F280342: 0x1F2E, + 0x1F290342: 0x1F2F, + 0x03B90313: 0x1F30, + 0x03B90314: 0x1F31, + 0x1F300300: 0x1F32, + 0x1F310300: 0x1F33, + 0x1F300301: 0x1F34, + 0x1F310301: 0x1F35, + 0x1F300342: 0x1F36, + 0x1F310342: 0x1F37, + 0x03990313: 0x1F38, + 0x03990314: 0x1F39, + 0x1F380300: 0x1F3A, + 0x1F390300: 0x1F3B, + 0x1F380301: 0x1F3C, + 0x1F390301: 0x1F3D, + 0x1F380342: 0x1F3E, + 0x1F390342: 0x1F3F, + 0x03BF0313: 0x1F40, + 0x03BF0314: 0x1F41, + 0x1F400300: 0x1F42, + 0x1F410300: 0x1F43, + 0x1F400301: 0x1F44, + 0x1F410301: 0x1F45, + 0x039F0313: 0x1F48, + 0x039F0314: 0x1F49, + 0x1F480300: 0x1F4A, + 0x1F490300: 0x1F4B, + 0x1F480301: 0x1F4C, + 0x1F490301: 0x1F4D, + 0x03C50313: 0x1F50, + 0x03C50314: 0x1F51, + 0x1F500300: 0x1F52, + 0x1F510300: 0x1F53, + 0x1F500301: 0x1F54, + 0x1F510301: 0x1F55, + 0x1F500342: 0x1F56, + 0x1F510342: 0x1F57, + 0x03A50314: 0x1F59, + 0x1F590300: 0x1F5B, + 0x1F590301: 0x1F5D, + 0x1F590342: 0x1F5F, + 0x03C90313: 0x1F60, + 0x03C90314: 0x1F61, + 0x1F600300: 0x1F62, + 0x1F610300: 0x1F63, + 0x1F600301: 0x1F64, + 0x1F610301: 0x1F65, + 0x1F600342: 0x1F66, + 0x1F610342: 0x1F67, + 0x03A90313: 0x1F68, + 0x03A90314: 0x1F69, + 0x1F680300: 0x1F6A, + 0x1F690300: 0x1F6B, + 0x1F680301: 0x1F6C, + 0x1F690301: 0x1F6D, + 0x1F680342: 0x1F6E, + 0x1F690342: 0x1F6F, + 0x03B10300: 0x1F70, + 0x03B50300: 0x1F72, + 0x03B70300: 0x1F74, + 0x03B90300: 0x1F76, + 0x03BF0300: 0x1F78, + 0x03C50300: 0x1F7A, + 0x03C90300: 0x1F7C, + 0x1F000345: 0x1F80, + 0x1F010345: 0x1F81, + 0x1F020345: 0x1F82, + 0x1F030345: 0x1F83, + 0x1F040345: 0x1F84, + 0x1F050345: 0x1F85, + 0x1F060345: 0x1F86, + 0x1F070345: 0x1F87, + 0x1F080345: 0x1F88, + 0x1F090345: 0x1F89, + 0x1F0A0345: 0x1F8A, + 0x1F0B0345: 0x1F8B, + 0x1F0C0345: 0x1F8C, + 0x1F0D0345: 0x1F8D, + 0x1F0E0345: 0x1F8E, + 0x1F0F0345: 0x1F8F, + 0x1F200345: 0x1F90, + 0x1F210345: 0x1F91, + 0x1F220345: 0x1F92, + 0x1F230345: 0x1F93, + 0x1F240345: 0x1F94, + 0x1F250345: 0x1F95, + 0x1F260345: 0x1F96, + 0x1F270345: 0x1F97, + 0x1F280345: 0x1F98, + 0x1F290345: 0x1F99, + 0x1F2A0345: 0x1F9A, + 0x1F2B0345: 0x1F9B, + 0x1F2C0345: 0x1F9C, + 0x1F2D0345: 0x1F9D, + 0x1F2E0345: 0x1F9E, + 0x1F2F0345: 0x1F9F, + 0x1F600345: 0x1FA0, + 0x1F610345: 0x1FA1, + 0x1F620345: 0x1FA2, + 0x1F630345: 0x1FA3, + 0x1F640345: 0x1FA4, + 0x1F650345: 0x1FA5, + 0x1F660345: 0x1FA6, + 0x1F670345: 0x1FA7, + 0x1F680345: 0x1FA8, + 0x1F690345: 0x1FA9, + 0x1F6A0345: 0x1FAA, + 0x1F6B0345: 0x1FAB, + 0x1F6C0345: 0x1FAC, + 0x1F6D0345: 0x1FAD, + 0x1F6E0345: 0x1FAE, + 0x1F6F0345: 0x1FAF, + 0x03B10306: 0x1FB0, + 0x03B10304: 0x1FB1, + 0x1F700345: 0x1FB2, + 0x03B10345: 0x1FB3, + 0x03AC0345: 0x1FB4, + 0x03B10342: 0x1FB6, + 0x1FB60345: 0x1FB7, + 0x03910306: 0x1FB8, + 0x03910304: 0x1FB9, + 0x03910300: 0x1FBA, + 0x03910345: 0x1FBC, + 0x00A80342: 0x1FC1, + 0x1F740345: 0x1FC2, + 0x03B70345: 0x1FC3, + 0x03AE0345: 0x1FC4, + 0x03B70342: 0x1FC6, + 0x1FC60345: 0x1FC7, + 0x03950300: 0x1FC8, + 0x03970300: 0x1FCA, + 0x03970345: 0x1FCC, + 0x1FBF0300: 0x1FCD, + 0x1FBF0301: 0x1FCE, + 0x1FBF0342: 0x1FCF, + 0x03B90306: 0x1FD0, + 0x03B90304: 0x1FD1, + 0x03CA0300: 0x1FD2, + 0x03B90342: 0x1FD6, + 0x03CA0342: 0x1FD7, + 0x03990306: 0x1FD8, + 0x03990304: 0x1FD9, + 0x03990300: 0x1FDA, + 0x1FFE0300: 0x1FDD, + 0x1FFE0301: 0x1FDE, + 0x1FFE0342: 0x1FDF, + 0x03C50306: 0x1FE0, + 0x03C50304: 0x1FE1, + 0x03CB0300: 0x1FE2, + 0x03C10313: 0x1FE4, + 0x03C10314: 0x1FE5, + 0x03C50342: 0x1FE6, + 0x03CB0342: 0x1FE7, + 0x03A50306: 0x1FE8, + 0x03A50304: 0x1FE9, + 0x03A50300: 0x1FEA, + 0x03A10314: 0x1FEC, + 0x00A80300: 0x1FED, + 0x1F7C0345: 0x1FF2, + 0x03C90345: 0x1FF3, + 0x03CE0345: 0x1FF4, + 0x03C90342: 0x1FF6, + 0x1FF60345: 0x1FF7, + 0x039F0300: 0x1FF8, + 0x03A90300: 0x1FFA, + 0x03A90345: 0x1FFC, + 0x21900338: 0x219A, + 0x21920338: 0x219B, + 0x21940338: 0x21AE, + 0x21D00338: 0x21CD, + 0x21D40338: 0x21CE, + 0x21D20338: 0x21CF, + 0x22030338: 0x2204, + 0x22080338: 0x2209, + 0x220B0338: 0x220C, + 0x22230338: 0x2224, + 0x22250338: 0x2226, + 0x223C0338: 0x2241, + 0x22430338: 0x2244, + 0x22450338: 0x2247, + 0x22480338: 0x2249, + 0x003D0338: 0x2260, + 0x22610338: 0x2262, + 0x224D0338: 0x226D, + 0x003C0338: 0x226E, + 0x003E0338: 0x226F, + 0x22640338: 0x2270, + 0x22650338: 0x2271, + 0x22720338: 0x2274, + 0x22730338: 0x2275, + 0x22760338: 0x2278, + 0x22770338: 0x2279, + 0x227A0338: 0x2280, + 0x227B0338: 0x2281, + 0x22820338: 0x2284, + 0x22830338: 0x2285, + 0x22860338: 0x2288, + 0x22870338: 0x2289, + 0x22A20338: 0x22AC, + 0x22A80338: 0x22AD, + 0x22A90338: 0x22AE, + 0x22AB0338: 0x22AF, + 0x227C0338: 0x22E0, + 0x227D0338: 0x22E1, + 0x22910338: 0x22E2, + 0x22920338: 0x22E3, + 0x22B20338: 0x22EA, + 0x22B30338: 0x22EB, + 0x22B40338: 0x22EC, + 0x22B50338: 0x22ED, + 0x304B3099: 0x304C, + 0x304D3099: 0x304E, + 0x304F3099: 0x3050, + 0x30513099: 0x3052, + 0x30533099: 0x3054, + 0x30553099: 0x3056, + 0x30573099: 0x3058, + 0x30593099: 0x305A, + 0x305B3099: 0x305C, + 0x305D3099: 0x305E, + 0x305F3099: 0x3060, + 0x30613099: 0x3062, + 0x30643099: 0x3065, + 0x30663099: 0x3067, + 0x30683099: 0x3069, + 0x306F3099: 0x3070, + 0x306F309A: 0x3071, + 0x30723099: 0x3073, + 0x3072309A: 0x3074, + 0x30753099: 0x3076, + 0x3075309A: 0x3077, + 0x30783099: 0x3079, + 0x3078309A: 0x307A, + 0x307B3099: 0x307C, + 0x307B309A: 0x307D, + 0x30463099: 0x3094, + 0x309D3099: 0x309E, + 0x30AB3099: 0x30AC, + 0x30AD3099: 0x30AE, + 0x30AF3099: 0x30B0, + 0x30B13099: 0x30B2, + 0x30B33099: 0x30B4, + 0x30B53099: 0x30B6, + 0x30B73099: 0x30B8, + 0x30B93099: 0x30BA, + 0x30BB3099: 0x30BC, + 0x30BD3099: 0x30BE, + 0x30BF3099: 0x30C0, + 0x30C13099: 0x30C2, + 0x30C43099: 0x30C5, + 0x30C63099: 0x30C7, + 0x30C83099: 0x30C9, + 0x30CF3099: 0x30D0, + 0x30CF309A: 0x30D1, + 0x30D23099: 0x30D3, + 0x30D2309A: 0x30D4, + 0x30D53099: 0x30D6, + 0x30D5309A: 0x30D7, + 0x30D83099: 0x30D9, + 0x30D8309A: 0x30DA, + 0x30DB3099: 0x30DC, + 0x30DB309A: 0x30DD, + 0x30A63099: 0x30F4, + 0x30EF3099: 0x30F7, + 0x30F03099: 0x30F8, + 0x30F13099: 0x30F9, + 0x30F23099: 0x30FA, + 0x30FD3099: 0x30FE, + 0x109910BA: 0x1109A, + 0x109B10BA: 0x1109C, + 0x10A510BA: 0x110AB, + 0x11311127: 0x1112E, + 0x11321127: 0x1112F, + 0x1347133E: 0x1134B, + 0x13471357: 0x1134C, + 0x14B914BA: 0x114BB, + 0x14B914B0: 0x114BC, + 0x14B914BD: 0x114BE, + 0x15B815AF: 0x115BA, + 0x15B915AF: 0x115BB, +} + +// Total size of tables: 53KB (53976 bytes) diff --git a/vendor/golang.org/x/text/unicode/norm/transform.go b/vendor/golang.org/x/text/unicode/norm/transform.go new file mode 100644 index 00000000000..8589067cde4 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/transform.go @@ -0,0 +1,88 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import ( + "unicode/utf8" + + "golang.org/x/text/transform" +) + +// Reset implements the Reset method of the transform.Transformer interface. +func (Form) Reset() {} + +// Transform implements the Transform method of the transform.Transformer +// interface. It may need to write segments of up to MaxSegmentSize at once. +// Users should either catch ErrShortDst and allow dst to grow or have dst be at +// least of size MaxTransformChunkSize to be guaranteed of progress. +func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + n := 0 + // Cap the maximum number of src bytes to check. + b := src + eof := atEOF + if ns := len(dst); ns < len(b) { + err = transform.ErrShortDst + eof = false + b = b[:ns] + } + i, ok := formTable[f].quickSpan(inputBytes(b), n, len(b), eof) + n += copy(dst[n:], b[n:i]) + if !ok { + nDst, nSrc, err = f.transform(dst[n:], src[n:], atEOF) + return nDst + n, nSrc + n, err + } + if n < len(src) && !atEOF { + err = transform.ErrShortSrc + } + return n, n, err +} + +func flushTransform(rb *reorderBuffer) bool { + // Write out (must fully fit in dst, or else it is a ErrShortDst). + if len(rb.out) < rb.nrune*utf8.UTFMax { + return false + } + rb.out = rb.out[rb.flushCopy(rb.out):] + return true +} + +var errs = []error{nil, transform.ErrShortDst, transform.ErrShortSrc} + +// transform implements the transform.Transformer interface. It is only called +// when quickSpan does not pass for a given string. +func (f Form) transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + // TODO: get rid of reorderBuffer. See CL 23460044. + rb := reorderBuffer{} + rb.init(f, src) + for { + // Load segment into reorder buffer. + rb.setFlusher(dst[nDst:], flushTransform) + end := decomposeSegment(&rb, nSrc, atEOF) + if end < 0 { + return nDst, nSrc, errs[-end] + } + nDst = len(dst) - len(rb.out) + nSrc = end + + // Next quickSpan. + end = rb.nsrc + eof := atEOF + if n := nSrc + len(dst) - nDst; n < end { + err = transform.ErrShortDst + end = n + eof = false + } + end, ok := rb.f.quickSpan(rb.src, nSrc, end, eof) + n := copy(dst[nDst:], rb.src.bytes[nSrc:end]) + nSrc += n + nDst += n + if ok { + if n < rb.nsrc && !atEOF { + err = transform.ErrShortSrc + } + return nDst, nSrc, err + } + } +} diff --git a/vendor/golang.org/x/text/unicode/norm/trie.go b/vendor/golang.org/x/text/unicode/norm/trie.go new file mode 100644 index 00000000000..423386bf436 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/trie.go @@ -0,0 +1,54 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +type valueRange struct { + value uint16 // header: value:stride + lo, hi byte // header: lo:n +} + +type sparseBlocks struct { + values []valueRange + offset []uint16 +} + +var nfcSparse = sparseBlocks{ + values: nfcSparseValues[:], + offset: nfcSparseOffset[:], +} + +var nfkcSparse = sparseBlocks{ + values: nfkcSparseValues[:], + offset: nfkcSparseOffset[:], +} + +var ( + nfcData = newNfcTrie(0) + nfkcData = newNfkcTrie(0) +) + +// lookupValue determines the type of block n and looks up the value for b. +// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block +// is a list of ranges with an accompanying value. Given a matching range r, +// the value for b is by r.value + (b - r.lo) * stride. +func (t *sparseBlocks) lookup(n uint32, b byte) uint16 { + offset := t.offset[n] + header := t.values[offset] + lo := offset + 1 + hi := lo + uint16(header.lo) + for lo < hi { + m := lo + (hi-lo)/2 + r := t.values[m] + if r.lo <= b && b <= r.hi { + return r.value + uint16(b-r.lo)*header.value + } + if b < r.lo { + hi = m + } else { + lo = m + 1 + } + } + return 0 +} diff --git a/vendor/golang.org/x/text/unicode/norm/triegen.go b/vendor/golang.org/x/text/unicode/norm/triegen.go new file mode 100644 index 00000000000..45d711900d1 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/triegen.go @@ -0,0 +1,117 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Trie table generator. +// Used by make*tables tools to generate a go file with trie data structures +// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte +// sequence are used to lookup offsets in the index table to be used for the +// next byte. The last byte is used to index into a table with 16-bit values. + +package main + +import ( + "fmt" + "io" +) + +const maxSparseEntries = 16 + +type normCompacter struct { + sparseBlocks [][]uint64 + sparseOffset []uint16 + sparseCount int + name string +} + +func mostFrequentStride(a []uint64) int { + counts := make(map[int]int) + var v int + for _, x := range a { + if stride := int(x) - v; v != 0 && stride >= 0 { + counts[stride]++ + } + v = int(x) + } + var maxs, maxc int + for stride, cnt := range counts { + if cnt > maxc || (cnt == maxc && stride < maxs) { + maxs, maxc = stride, cnt + } + } + return maxs +} + +func countSparseEntries(a []uint64) int { + stride := mostFrequentStride(a) + var v, count int + for _, tv := range a { + if int(tv)-v != stride { + if tv != 0 { + count++ + } + } + v = int(tv) + } + return count +} + +func (c *normCompacter) Size(v []uint64) (sz int, ok bool) { + if n := countSparseEntries(v); n <= maxSparseEntries { + return (n+1)*4 + 2, true + } + return 0, false +} + +func (c *normCompacter) Store(v []uint64) uint32 { + h := uint32(len(c.sparseOffset)) + c.sparseBlocks = append(c.sparseBlocks, v) + c.sparseOffset = append(c.sparseOffset, uint16(c.sparseCount)) + c.sparseCount += countSparseEntries(v) + 1 + return h +} + +func (c *normCompacter) Handler() string { + return c.name + "Sparse.lookup" +} + +func (c *normCompacter) Print(w io.Writer) (retErr error) { + p := func(f string, x ...interface{}) { + if _, err := fmt.Fprintf(w, f, x...); retErr == nil && err != nil { + retErr = err + } + } + + ls := len(c.sparseBlocks) + p("// %sSparseOffset: %d entries, %d bytes\n", c.name, ls, ls*2) + p("var %sSparseOffset = %#v\n\n", c.name, c.sparseOffset) + + ns := c.sparseCount + p("// %sSparseValues: %d entries, %d bytes\n", c.name, ns, ns*4) + p("var %sSparseValues = [%d]valueRange {", c.name, ns) + for i, b := range c.sparseBlocks { + p("\n// Block %#x, offset %#x", i, c.sparseOffset[i]) + var v int + stride := mostFrequentStride(b) + n := countSparseEntries(b) + p("\n{value:%#04x,lo:%#02x},", stride, uint8(n)) + for i, nv := range b { + if int(nv)-v != stride { + if v != 0 { + p(",hi:%#02x},", 0x80+i-1) + } + if nv != 0 { + p("\n{value:%#04x,lo:%#02x", nv, 0x80+i) + } + } + v = int(nv) + } + if v != 0 { + p(",hi:%#02x},", 0x80+len(b)-1) + } + } + p("\n}\n\n") + return +} diff --git a/vendor/google.golang.org/genproto/LICENSE b/vendor/google.golang.org/genproto/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/vendor/google.golang.org/genproto/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go b/vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go new file mode 100644 index 00000000000..fb104e6dfb8 --- /dev/null +++ b/vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go @@ -0,0 +1,156 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: google/rpc/status.proto + +package status // import "google.golang.org/genproto/googleapis/rpc/status" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import any "github.com/golang/protobuf/ptypes/any" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// The `Status` type defines a logical error model that is suitable for different +// programming environments, including REST APIs and RPC APIs. It is used by +// [gRPC](https://github.com/grpc). The error model is designed to be: +// +// - Simple to use and understand for most users +// - Flexible enough to meet unexpected needs +// +// # Overview +// +// The `Status` message contains three pieces of data: error code, error message, +// and error details. The error code should be an enum value of +// [google.rpc.Code][google.rpc.Code], but it may accept additional error codes if needed. The +// error message should be a developer-facing English message that helps +// developers *understand* and *resolve* the error. If a localized user-facing +// error message is needed, put the localized message in the error details or +// localize it in the client. The optional error details may contain arbitrary +// information about the error. There is a predefined set of error detail types +// in the package `google.rpc` that can be used for common error conditions. +// +// # Language mapping +// +// The `Status` message is the logical representation of the error model, but it +// is not necessarily the actual wire format. When the `Status` message is +// exposed in different client libraries and different wire protocols, it can be +// mapped differently. For example, it will likely be mapped to some exceptions +// in Java, but more likely mapped to some error codes in C. +// +// # Other uses +// +// The error model and the `Status` message can be used in a variety of +// environments, either with or without APIs, to provide a +// consistent developer experience across different environments. +// +// Example uses of this error model include: +// +// - Partial errors. If a service needs to return partial errors to the client, +// it may embed the `Status` in the normal response to indicate the partial +// errors. +// +// - Workflow errors. A typical workflow has multiple steps. Each step may +// have a `Status` message for error reporting. +// +// - Batch operations. If a client uses batch request and batch response, the +// `Status` message should be used directly inside batch response, one for +// each error sub-response. +// +// - Asynchronous operations. If an API call embeds asynchronous operation +// results in its response, the status of those operations should be +// represented directly using the `Status` message. +// +// - Logging. If some API errors are stored in logs, the message `Status` could +// be used directly after any stripping needed for security/privacy reasons. +type Status struct { + // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + Code int32 `protobuf:"varint,1,opt,name=code" json:"code,omitempty"` + // A developer-facing error message, which should be in English. Any + // user-facing error message should be localized and sent in the + // [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. + Message string `protobuf:"bytes,2,opt,name=message" json:"message,omitempty"` + // A list of messages that carry the error details. There is a common set of + // message types for APIs to use. + Details []*any.Any `protobuf:"bytes,3,rep,name=details" json:"details,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Status) Reset() { *m = Status{} } +func (m *Status) String() string { return proto.CompactTextString(m) } +func (*Status) ProtoMessage() {} +func (*Status) Descriptor() ([]byte, []int) { + return fileDescriptor_status_c656c685916bdf47, []int{0} +} +func (m *Status) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Status.Unmarshal(m, b) +} +func (m *Status) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Status.Marshal(b, m, deterministic) +} +func (dst *Status) XXX_Merge(src proto.Message) { + xxx_messageInfo_Status.Merge(dst, src) +} +func (m *Status) XXX_Size() int { + return xxx_messageInfo_Status.Size(m) +} +func (m *Status) XXX_DiscardUnknown() { + xxx_messageInfo_Status.DiscardUnknown(m) +} + +var xxx_messageInfo_Status proto.InternalMessageInfo + +func (m *Status) GetCode() int32 { + if m != nil { + return m.Code + } + return 0 +} + +func (m *Status) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +func (m *Status) GetDetails() []*any.Any { + if m != nil { + return m.Details + } + return nil +} + +func init() { + proto.RegisterType((*Status)(nil), "google.rpc.Status") +} + +func init() { proto.RegisterFile("google/rpc/status.proto", fileDescriptor_status_c656c685916bdf47) } + +var fileDescriptor_status_c656c685916bdf47 = []byte{ + // 209 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4f, 0xcf, 0xcf, 0x4f, + 0xcf, 0x49, 0xd5, 0x2f, 0x2a, 0x48, 0xd6, 0x2f, 0x2e, 0x49, 0x2c, 0x29, 0x2d, 0xd6, 0x2b, 0x28, + 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x82, 0x48, 0xe8, 0x15, 0x15, 0x24, 0x4b, 0x49, 0x42, 0x15, 0x81, + 0x65, 0x92, 0x4a, 0xd3, 0xf4, 0x13, 0xf3, 0x2a, 0x21, 0xca, 0x94, 0xd2, 0xb8, 0xd8, 0x82, 0xc1, + 0xda, 0x84, 0x84, 0xb8, 0x58, 0x92, 0xf3, 0x53, 0x52, 0x25, 0x18, 0x15, 0x18, 0x35, 0x58, 0x83, + 0xc0, 0x6c, 0x21, 0x09, 0x2e, 0xf6, 0xdc, 0xd4, 0xe2, 0xe2, 0xc4, 0xf4, 0x54, 0x09, 0x26, 0x05, + 0x46, 0x0d, 0xce, 0x20, 0x18, 0x57, 0x48, 0x8f, 0x8b, 0x3d, 0x25, 0xb5, 0x24, 0x31, 0x33, 0xa7, + 0x58, 0x82, 0x59, 0x81, 0x59, 0x83, 0xdb, 0x48, 0x44, 0x0f, 0x6a, 0x21, 0xcc, 0x12, 0x3d, 0xc7, + 0xbc, 0xca, 0x20, 0x98, 0x22, 0xa7, 0x38, 0x2e, 0xbe, 0xe4, 0xfc, 0x5c, 0x3d, 0x84, 0xa3, 0x9c, + 0xb8, 0x21, 0xf6, 0x06, 0x80, 0x94, 0x07, 0x30, 0x46, 0x99, 0x43, 0xa5, 0xd2, 0xf3, 0x73, 0x12, + 0xf3, 0xd2, 0xf5, 0xf2, 0x8b, 0xd2, 0xf5, 0xd3, 0x53, 0xf3, 0xc0, 0x86, 0xe9, 0x43, 0xa4, 0x12, + 0x0b, 0x32, 0x8b, 0x91, 0xfc, 0x69, 0x0d, 0xa1, 0x16, 0x31, 0x31, 0x07, 0x05, 0x38, 0x27, 0xb1, + 0x81, 0x55, 0x1a, 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, 0xa4, 0x53, 0xf0, 0x7c, 0x10, 0x01, 0x00, + 0x00, +} diff --git a/vendor/google.golang.org/grpc/balancer/base/balancer.go b/vendor/google.golang.org/grpc/balancer/base/balancer.go new file mode 100644 index 00000000000..23d13511bb2 --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer/base/balancer.go @@ -0,0 +1,208 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package base + +import ( + "golang.org/x/net/context" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" +) + +type baseBuilder struct { + name string + pickerBuilder PickerBuilder +} + +func (bb *baseBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { + return &baseBalancer{ + cc: cc, + pickerBuilder: bb.pickerBuilder, + + subConns: make(map[resolver.Address]balancer.SubConn), + scStates: make(map[balancer.SubConn]connectivity.State), + csEvltr: &connectivityStateEvaluator{}, + // Initialize picker to a picker that always return + // ErrNoSubConnAvailable, because when state of a SubConn changes, we + // may call UpdateBalancerState with this picker. + picker: NewErrPicker(balancer.ErrNoSubConnAvailable), + } +} + +func (bb *baseBuilder) Name() string { + return bb.name +} + +type baseBalancer struct { + cc balancer.ClientConn + pickerBuilder PickerBuilder + + csEvltr *connectivityStateEvaluator + state connectivity.State + + subConns map[resolver.Address]balancer.SubConn + scStates map[balancer.SubConn]connectivity.State + picker balancer.Picker +} + +func (b *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) { + if err != nil { + grpclog.Infof("base.baseBalancer: HandleResolvedAddrs called with error %v", err) + return + } + grpclog.Infoln("base.baseBalancer: got new resolved addresses: ", addrs) + // addrsSet is the set converted from addrs, it's used for quick lookup of an address. + addrsSet := make(map[resolver.Address]struct{}) + for _, a := range addrs { + addrsSet[a] = struct{}{} + if _, ok := b.subConns[a]; !ok { + // a is a new address (not existing in b.subConns). + sc, err := b.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{}) + if err != nil { + grpclog.Warningf("base.baseBalancer: failed to create new SubConn: %v", err) + continue + } + b.subConns[a] = sc + b.scStates[sc] = connectivity.Idle + sc.Connect() + } + } + for a, sc := range b.subConns { + // a was removed by resolver. + if _, ok := addrsSet[a]; !ok { + b.cc.RemoveSubConn(sc) + delete(b.subConns, a) + // Keep the state of this sc in b.scStates until sc's state becomes Shutdown. + // The entry will be deleted in HandleSubConnStateChange. + } + } +} + +// regeneratePicker takes a snapshot of the balancer, and generates a picker +// from it. The picker is +// - errPicker with ErrTransientFailure if the balancer is in TransientFailure, +// - built by the pickerBuilder with all READY SubConns otherwise. +func (b *baseBalancer) regeneratePicker() { + if b.state == connectivity.TransientFailure { + b.picker = NewErrPicker(balancer.ErrTransientFailure) + return + } + readySCs := make(map[resolver.Address]balancer.SubConn) + + // Filter out all ready SCs from full subConn map. + for addr, sc := range b.subConns { + if st, ok := b.scStates[sc]; ok && st == connectivity.Ready { + readySCs[addr] = sc + } + } + b.picker = b.pickerBuilder.Build(readySCs) +} + +func (b *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { + grpclog.Infof("base.baseBalancer: handle SubConn state change: %p, %v", sc, s) + oldS, ok := b.scStates[sc] + if !ok { + grpclog.Infof("base.baseBalancer: got state changes for an unknown SubConn: %p, %v", sc, s) + return + } + b.scStates[sc] = s + switch s { + case connectivity.Idle: + sc.Connect() + case connectivity.Shutdown: + // When an address was removed by resolver, b called RemoveSubConn but + // kept the sc's state in scStates. Remove state for this sc here. + delete(b.scStates, sc) + } + + oldAggrState := b.state + b.state = b.csEvltr.recordTransition(oldS, s) + + // Regenerate picker when one of the following happens: + // - this sc became ready from not-ready + // - this sc became not-ready from ready + // - the aggregated state of balancer became TransientFailure from non-TransientFailure + // - the aggregated state of balancer became non-TransientFailure from TransientFailure + if (s == connectivity.Ready) != (oldS == connectivity.Ready) || + (b.state == connectivity.TransientFailure) != (oldAggrState == connectivity.TransientFailure) { + b.regeneratePicker() + } + + b.cc.UpdateBalancerState(b.state, b.picker) +} + +// Close is a nop because base balancer doesn't have internal state to clean up, +// and it doesn't need to call RemoveSubConn for the SubConns. +func (b *baseBalancer) Close() { +} + +// NewErrPicker returns a picker that always returns err on Pick(). +func NewErrPicker(err error) balancer.Picker { + return &errPicker{err: err} +} + +type errPicker struct { + err error // Pick() always returns this err. +} + +func (p *errPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { + return nil, nil, p.err +} + +// connectivityStateEvaluator gets updated by addrConns when their +// states transition, based on which it evaluates the state of +// ClientConn. +type connectivityStateEvaluator struct { + numReady uint64 // Number of addrConns in ready state. + numConnecting uint64 // Number of addrConns in connecting state. + numTransientFailure uint64 // Number of addrConns in transientFailure. +} + +// recordTransition records state change happening in every subConn and based on +// that it evaluates what aggregated state should be. +// It can only transition between Ready, Connecting and TransientFailure. Other states, +// Idle and Shutdown are transitioned into by ClientConn; in the beginning of the connection +// before any subConn is created ClientConn is in idle state. In the end when ClientConn +// closes it is in Shutdown state. +// +// recordTransition should only be called synchronously from the same goroutine. +func (cse *connectivityStateEvaluator) recordTransition(oldState, newState connectivity.State) connectivity.State { + // Update counters. + for idx, state := range []connectivity.State{oldState, newState} { + updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. + switch state { + case connectivity.Ready: + cse.numReady += updateVal + case connectivity.Connecting: + cse.numConnecting += updateVal + case connectivity.TransientFailure: + cse.numTransientFailure += updateVal + } + } + + // Evaluate. + if cse.numReady > 0 { + return connectivity.Ready + } + if cse.numConnecting > 0 { + return connectivity.Connecting + } + return connectivity.TransientFailure +} diff --git a/vendor/google.golang.org/grpc/balancer/base/base.go b/vendor/google.golang.org/grpc/balancer/base/base.go new file mode 100644 index 00000000000..012ace2f2f7 --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer/base/base.go @@ -0,0 +1,52 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package base defines a balancer base that can be used to build balancers with +// different picking algorithms. +// +// The base balancer creates a new SubConn for each resolved address. The +// provided picker will only be notified about READY SubConns. +// +// This package is the base of round_robin balancer, its purpose is to be used +// to build round_robin like balancers with complex picking algorithms. +// Balancers with more complicated logic should try to implement a balancer +// builder from scratch. +// +// All APIs in this package are experimental. +package base + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" +) + +// PickerBuilder creates balancer.Picker. +type PickerBuilder interface { + // Build takes a slice of ready SubConns, and returns a picker that will be + // used by gRPC to pick a SubConn. + Build(readySCs map[resolver.Address]balancer.SubConn) balancer.Picker +} + +// NewBalancerBuilder returns a balancer builder. The balancers +// built by this builder will use the picker builder to build pickers. +func NewBalancerBuilder(name string, pb PickerBuilder) balancer.Builder { + return &baseBuilder{ + name: name, + pickerBuilder: pb, + } +} diff --git a/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go b/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go new file mode 100644 index 00000000000..2eda0a1c210 --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go @@ -0,0 +1,79 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package roundrobin defines a roundrobin balancer. Roundrobin balancer is +// installed as one of the default balancers in gRPC, users don't need to +// explicitly install this balancer. +package roundrobin + +import ( + "sync" + + "golang.org/x/net/context" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" +) + +// Name is the name of round_robin balancer. +const Name = "round_robin" + +// newBuilder creates a new roundrobin balancer builder. +func newBuilder() balancer.Builder { + return base.NewBalancerBuilder(Name, &rrPickerBuilder{}) +} + +func init() { + balancer.Register(newBuilder()) +} + +type rrPickerBuilder struct{} + +func (*rrPickerBuilder) Build(readySCs map[resolver.Address]balancer.SubConn) balancer.Picker { + grpclog.Infof("roundrobinPicker: newPicker called with readySCs: %v", readySCs) + var scs []balancer.SubConn + for _, sc := range readySCs { + scs = append(scs, sc) + } + return &rrPicker{ + subConns: scs, + } +} + +type rrPicker struct { + // subConns is the snapshot of the roundrobin balancer when this picker was + // created. The slice is immutable. Each Get() will do a round robin + // selection from it and return the selected SubConn. + subConns []balancer.SubConn + + mu sync.Mutex + next int +} + +func (p *rrPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { + if len(p.subConns) <= 0 { + return nil, nil, balancer.ErrNoSubConnAvailable + } + + p.mu.Lock() + sc := p.subConns[p.next] + p.next = (p.next + 1) % len(p.subConns) + p.mu.Unlock() + return sc, nil, nil +} diff --git a/vendor/google.golang.org/grpc/connectivity/connectivity.go b/vendor/google.golang.org/grpc/connectivity/connectivity.go new file mode 100644 index 00000000000..568ef5dc68b --- /dev/null +++ b/vendor/google.golang.org/grpc/connectivity/connectivity.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package connectivity defines connectivity semantics. +// For details, see https://github.com/grpc/grpc/blob/master/doc/connectivity-semantics-and-api.md. +// All APIs in this package are experimental. +package connectivity + +import ( + "golang.org/x/net/context" + "google.golang.org/grpc/grpclog" +) + +// State indicates the state of connectivity. +// It can be the state of a ClientConn or SubConn. +type State int + +func (s State) String() string { + switch s { + case Idle: + return "IDLE" + case Connecting: + return "CONNECTING" + case Ready: + return "READY" + case TransientFailure: + return "TRANSIENT_FAILURE" + case Shutdown: + return "SHUTDOWN" + default: + grpclog.Errorf("unknown connectivity state: %d", s) + return "Invalid-State" + } +} + +const ( + // Idle indicates the ClientConn is idle. + Idle State = iota + // Connecting indicates the ClienConn is connecting. + Connecting + // Ready indicates the ClientConn is ready for work. + Ready + // TransientFailure indicates the ClientConn has seen a failure but expects to recover. + TransientFailure + // Shutdown indicates the ClientConn has started shutting down. + Shutdown +) + +// Reporter reports the connectivity states. +type Reporter interface { + // CurrentState returns the current state of the reporter. + CurrentState() State + // WaitForStateChange blocks until the reporter's state is different from the given state, + // and returns true. + // It returns false if <-ctx.Done() can proceed (ctx got timeout or got canceled). + WaitForStateChange(context.Context, State) bool +} diff --git a/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.pb.go b/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.pb.go new file mode 100644 index 00000000000..b3b32b48e86 --- /dev/null +++ b/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.pb.go @@ -0,0 +1,799 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: grpc_lb_v1/messages/messages.proto + +package messages // import "google.golang.org/grpc/grpclb/grpc_lb_v1/messages" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type Duration struct { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. + Seconds int64 `protobuf:"varint,1,opt,name=seconds" json:"seconds,omitempty"` + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + Nanos int32 `protobuf:"varint,2,opt,name=nanos" json:"nanos,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Duration) Reset() { *m = Duration{} } +func (m *Duration) String() string { return proto.CompactTextString(m) } +func (*Duration) ProtoMessage() {} +func (*Duration) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b81c731f0e83edbd, []int{0} +} +func (m *Duration) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Duration.Unmarshal(m, b) +} +func (m *Duration) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Duration.Marshal(b, m, deterministic) +} +func (dst *Duration) XXX_Merge(src proto.Message) { + xxx_messageInfo_Duration.Merge(dst, src) +} +func (m *Duration) XXX_Size() int { + return xxx_messageInfo_Duration.Size(m) +} +func (m *Duration) XXX_DiscardUnknown() { + xxx_messageInfo_Duration.DiscardUnknown(m) +} + +var xxx_messageInfo_Duration proto.InternalMessageInfo + +func (m *Duration) GetSeconds() int64 { + if m != nil { + return m.Seconds + } + return 0 +} + +func (m *Duration) GetNanos() int32 { + if m != nil { + return m.Nanos + } + return 0 +} + +type Timestamp struct { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + Seconds int64 `protobuf:"varint,1,opt,name=seconds" json:"seconds,omitempty"` + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + Nanos int32 `protobuf:"varint,2,opt,name=nanos" json:"nanos,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Timestamp) Reset() { *m = Timestamp{} } +func (m *Timestamp) String() string { return proto.CompactTextString(m) } +func (*Timestamp) ProtoMessage() {} +func (*Timestamp) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b81c731f0e83edbd, []int{1} +} +func (m *Timestamp) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Timestamp.Unmarshal(m, b) +} +func (m *Timestamp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Timestamp.Marshal(b, m, deterministic) +} +func (dst *Timestamp) XXX_Merge(src proto.Message) { + xxx_messageInfo_Timestamp.Merge(dst, src) +} +func (m *Timestamp) XXX_Size() int { + return xxx_messageInfo_Timestamp.Size(m) +} +func (m *Timestamp) XXX_DiscardUnknown() { + xxx_messageInfo_Timestamp.DiscardUnknown(m) +} + +var xxx_messageInfo_Timestamp proto.InternalMessageInfo + +func (m *Timestamp) GetSeconds() int64 { + if m != nil { + return m.Seconds + } + return 0 +} + +func (m *Timestamp) GetNanos() int32 { + if m != nil { + return m.Nanos + } + return 0 +} + +type LoadBalanceRequest struct { + // Types that are valid to be assigned to LoadBalanceRequestType: + // *LoadBalanceRequest_InitialRequest + // *LoadBalanceRequest_ClientStats + LoadBalanceRequestType isLoadBalanceRequest_LoadBalanceRequestType `protobuf_oneof:"load_balance_request_type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *LoadBalanceRequest) Reset() { *m = LoadBalanceRequest{} } +func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) } +func (*LoadBalanceRequest) ProtoMessage() {} +func (*LoadBalanceRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b81c731f0e83edbd, []int{2} +} +func (m *LoadBalanceRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoadBalanceRequest.Unmarshal(m, b) +} +func (m *LoadBalanceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoadBalanceRequest.Marshal(b, m, deterministic) +} +func (dst *LoadBalanceRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoadBalanceRequest.Merge(dst, src) +} +func (m *LoadBalanceRequest) XXX_Size() int { + return xxx_messageInfo_LoadBalanceRequest.Size(m) +} +func (m *LoadBalanceRequest) XXX_DiscardUnknown() { + xxx_messageInfo_LoadBalanceRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_LoadBalanceRequest proto.InternalMessageInfo + +type isLoadBalanceRequest_LoadBalanceRequestType interface { + isLoadBalanceRequest_LoadBalanceRequestType() +} + +type LoadBalanceRequest_InitialRequest struct { + InitialRequest *InitialLoadBalanceRequest `protobuf:"bytes,1,opt,name=initial_request,json=initialRequest,oneof"` +} +type LoadBalanceRequest_ClientStats struct { + ClientStats *ClientStats `protobuf:"bytes,2,opt,name=client_stats,json=clientStats,oneof"` +} + +func (*LoadBalanceRequest_InitialRequest) isLoadBalanceRequest_LoadBalanceRequestType() {} +func (*LoadBalanceRequest_ClientStats) isLoadBalanceRequest_LoadBalanceRequestType() {} + +func (m *LoadBalanceRequest) GetLoadBalanceRequestType() isLoadBalanceRequest_LoadBalanceRequestType { + if m != nil { + return m.LoadBalanceRequestType + } + return nil +} + +func (m *LoadBalanceRequest) GetInitialRequest() *InitialLoadBalanceRequest { + if x, ok := m.GetLoadBalanceRequestType().(*LoadBalanceRequest_InitialRequest); ok { + return x.InitialRequest + } + return nil +} + +func (m *LoadBalanceRequest) GetClientStats() *ClientStats { + if x, ok := m.GetLoadBalanceRequestType().(*LoadBalanceRequest_ClientStats); ok { + return x.ClientStats + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*LoadBalanceRequest) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _LoadBalanceRequest_OneofMarshaler, _LoadBalanceRequest_OneofUnmarshaler, _LoadBalanceRequest_OneofSizer, []interface{}{ + (*LoadBalanceRequest_InitialRequest)(nil), + (*LoadBalanceRequest_ClientStats)(nil), + } +} + +func _LoadBalanceRequest_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*LoadBalanceRequest) + // load_balance_request_type + switch x := m.LoadBalanceRequestType.(type) { + case *LoadBalanceRequest_InitialRequest: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.InitialRequest); err != nil { + return err + } + case *LoadBalanceRequest_ClientStats: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.ClientStats); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("LoadBalanceRequest.LoadBalanceRequestType has unexpected type %T", x) + } + return nil +} + +func _LoadBalanceRequest_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*LoadBalanceRequest) + switch tag { + case 1: // load_balance_request_type.initial_request + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(InitialLoadBalanceRequest) + err := b.DecodeMessage(msg) + m.LoadBalanceRequestType = &LoadBalanceRequest_InitialRequest{msg} + return true, err + case 2: // load_balance_request_type.client_stats + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ClientStats) + err := b.DecodeMessage(msg) + m.LoadBalanceRequestType = &LoadBalanceRequest_ClientStats{msg} + return true, err + default: + return false, nil + } +} + +func _LoadBalanceRequest_OneofSizer(msg proto.Message) (n int) { + m := msg.(*LoadBalanceRequest) + // load_balance_request_type + switch x := m.LoadBalanceRequestType.(type) { + case *LoadBalanceRequest_InitialRequest: + s := proto.Size(x.InitialRequest) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *LoadBalanceRequest_ClientStats: + s := proto.Size(x.ClientStats) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +type InitialLoadBalanceRequest struct { + // Name of load balanced service (IE, balancer.service.com) + // length should be less than 256 bytes. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *InitialLoadBalanceRequest) Reset() { *m = InitialLoadBalanceRequest{} } +func (m *InitialLoadBalanceRequest) String() string { return proto.CompactTextString(m) } +func (*InitialLoadBalanceRequest) ProtoMessage() {} +func (*InitialLoadBalanceRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b81c731f0e83edbd, []int{3} +} +func (m *InitialLoadBalanceRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_InitialLoadBalanceRequest.Unmarshal(m, b) +} +func (m *InitialLoadBalanceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_InitialLoadBalanceRequest.Marshal(b, m, deterministic) +} +func (dst *InitialLoadBalanceRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_InitialLoadBalanceRequest.Merge(dst, src) +} +func (m *InitialLoadBalanceRequest) XXX_Size() int { + return xxx_messageInfo_InitialLoadBalanceRequest.Size(m) +} +func (m *InitialLoadBalanceRequest) XXX_DiscardUnknown() { + xxx_messageInfo_InitialLoadBalanceRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_InitialLoadBalanceRequest proto.InternalMessageInfo + +func (m *InitialLoadBalanceRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +// Contains client level statistics that are useful to load balancing. Each +// count except the timestamp should be reset to zero after reporting the stats. +type ClientStats struct { + // The timestamp of generating the report. + Timestamp *Timestamp `protobuf:"bytes,1,opt,name=timestamp" json:"timestamp,omitempty"` + // The total number of RPCs that started. + NumCallsStarted int64 `protobuf:"varint,2,opt,name=num_calls_started,json=numCallsStarted" json:"num_calls_started,omitempty"` + // The total number of RPCs that finished. + NumCallsFinished int64 `protobuf:"varint,3,opt,name=num_calls_finished,json=numCallsFinished" json:"num_calls_finished,omitempty"` + // The total number of RPCs that were dropped by the client because of rate + // limiting. + NumCallsFinishedWithDropForRateLimiting int64 `protobuf:"varint,4,opt,name=num_calls_finished_with_drop_for_rate_limiting,json=numCallsFinishedWithDropForRateLimiting" json:"num_calls_finished_with_drop_for_rate_limiting,omitempty"` + // The total number of RPCs that were dropped by the client because of load + // balancing. + NumCallsFinishedWithDropForLoadBalancing int64 `protobuf:"varint,5,opt,name=num_calls_finished_with_drop_for_load_balancing,json=numCallsFinishedWithDropForLoadBalancing" json:"num_calls_finished_with_drop_for_load_balancing,omitempty"` + // The total number of RPCs that failed to reach a server except dropped RPCs. + NumCallsFinishedWithClientFailedToSend int64 `protobuf:"varint,6,opt,name=num_calls_finished_with_client_failed_to_send,json=numCallsFinishedWithClientFailedToSend" json:"num_calls_finished_with_client_failed_to_send,omitempty"` + // The total number of RPCs that finished and are known to have been received + // by a server. + NumCallsFinishedKnownReceived int64 `protobuf:"varint,7,opt,name=num_calls_finished_known_received,json=numCallsFinishedKnownReceived" json:"num_calls_finished_known_received,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ClientStats) Reset() { *m = ClientStats{} } +func (m *ClientStats) String() string { return proto.CompactTextString(m) } +func (*ClientStats) ProtoMessage() {} +func (*ClientStats) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b81c731f0e83edbd, []int{4} +} +func (m *ClientStats) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ClientStats.Unmarshal(m, b) +} +func (m *ClientStats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ClientStats.Marshal(b, m, deterministic) +} +func (dst *ClientStats) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClientStats.Merge(dst, src) +} +func (m *ClientStats) XXX_Size() int { + return xxx_messageInfo_ClientStats.Size(m) +} +func (m *ClientStats) XXX_DiscardUnknown() { + xxx_messageInfo_ClientStats.DiscardUnknown(m) +} + +var xxx_messageInfo_ClientStats proto.InternalMessageInfo + +func (m *ClientStats) GetTimestamp() *Timestamp { + if m != nil { + return m.Timestamp + } + return nil +} + +func (m *ClientStats) GetNumCallsStarted() int64 { + if m != nil { + return m.NumCallsStarted + } + return 0 +} + +func (m *ClientStats) GetNumCallsFinished() int64 { + if m != nil { + return m.NumCallsFinished + } + return 0 +} + +func (m *ClientStats) GetNumCallsFinishedWithDropForRateLimiting() int64 { + if m != nil { + return m.NumCallsFinishedWithDropForRateLimiting + } + return 0 +} + +func (m *ClientStats) GetNumCallsFinishedWithDropForLoadBalancing() int64 { + if m != nil { + return m.NumCallsFinishedWithDropForLoadBalancing + } + return 0 +} + +func (m *ClientStats) GetNumCallsFinishedWithClientFailedToSend() int64 { + if m != nil { + return m.NumCallsFinishedWithClientFailedToSend + } + return 0 +} + +func (m *ClientStats) GetNumCallsFinishedKnownReceived() int64 { + if m != nil { + return m.NumCallsFinishedKnownReceived + } + return 0 +} + +type LoadBalanceResponse struct { + // Types that are valid to be assigned to LoadBalanceResponseType: + // *LoadBalanceResponse_InitialResponse + // *LoadBalanceResponse_ServerList + LoadBalanceResponseType isLoadBalanceResponse_LoadBalanceResponseType `protobuf_oneof:"load_balance_response_type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *LoadBalanceResponse) Reset() { *m = LoadBalanceResponse{} } +func (m *LoadBalanceResponse) String() string { return proto.CompactTextString(m) } +func (*LoadBalanceResponse) ProtoMessage() {} +func (*LoadBalanceResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b81c731f0e83edbd, []int{5} +} +func (m *LoadBalanceResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoadBalanceResponse.Unmarshal(m, b) +} +func (m *LoadBalanceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoadBalanceResponse.Marshal(b, m, deterministic) +} +func (dst *LoadBalanceResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoadBalanceResponse.Merge(dst, src) +} +func (m *LoadBalanceResponse) XXX_Size() int { + return xxx_messageInfo_LoadBalanceResponse.Size(m) +} +func (m *LoadBalanceResponse) XXX_DiscardUnknown() { + xxx_messageInfo_LoadBalanceResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_LoadBalanceResponse proto.InternalMessageInfo + +type isLoadBalanceResponse_LoadBalanceResponseType interface { + isLoadBalanceResponse_LoadBalanceResponseType() +} + +type LoadBalanceResponse_InitialResponse struct { + InitialResponse *InitialLoadBalanceResponse `protobuf:"bytes,1,opt,name=initial_response,json=initialResponse,oneof"` +} +type LoadBalanceResponse_ServerList struct { + ServerList *ServerList `protobuf:"bytes,2,opt,name=server_list,json=serverList,oneof"` +} + +func (*LoadBalanceResponse_InitialResponse) isLoadBalanceResponse_LoadBalanceResponseType() {} +func (*LoadBalanceResponse_ServerList) isLoadBalanceResponse_LoadBalanceResponseType() {} + +func (m *LoadBalanceResponse) GetLoadBalanceResponseType() isLoadBalanceResponse_LoadBalanceResponseType { + if m != nil { + return m.LoadBalanceResponseType + } + return nil +} + +func (m *LoadBalanceResponse) GetInitialResponse() *InitialLoadBalanceResponse { + if x, ok := m.GetLoadBalanceResponseType().(*LoadBalanceResponse_InitialResponse); ok { + return x.InitialResponse + } + return nil +} + +func (m *LoadBalanceResponse) GetServerList() *ServerList { + if x, ok := m.GetLoadBalanceResponseType().(*LoadBalanceResponse_ServerList); ok { + return x.ServerList + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*LoadBalanceResponse) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _LoadBalanceResponse_OneofMarshaler, _LoadBalanceResponse_OneofUnmarshaler, _LoadBalanceResponse_OneofSizer, []interface{}{ + (*LoadBalanceResponse_InitialResponse)(nil), + (*LoadBalanceResponse_ServerList)(nil), + } +} + +func _LoadBalanceResponse_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*LoadBalanceResponse) + // load_balance_response_type + switch x := m.LoadBalanceResponseType.(type) { + case *LoadBalanceResponse_InitialResponse: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.InitialResponse); err != nil { + return err + } + case *LoadBalanceResponse_ServerList: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.ServerList); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("LoadBalanceResponse.LoadBalanceResponseType has unexpected type %T", x) + } + return nil +} + +func _LoadBalanceResponse_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*LoadBalanceResponse) + switch tag { + case 1: // load_balance_response_type.initial_response + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(InitialLoadBalanceResponse) + err := b.DecodeMessage(msg) + m.LoadBalanceResponseType = &LoadBalanceResponse_InitialResponse{msg} + return true, err + case 2: // load_balance_response_type.server_list + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ServerList) + err := b.DecodeMessage(msg) + m.LoadBalanceResponseType = &LoadBalanceResponse_ServerList{msg} + return true, err + default: + return false, nil + } +} + +func _LoadBalanceResponse_OneofSizer(msg proto.Message) (n int) { + m := msg.(*LoadBalanceResponse) + // load_balance_response_type + switch x := m.LoadBalanceResponseType.(type) { + case *LoadBalanceResponse_InitialResponse: + s := proto.Size(x.InitialResponse) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *LoadBalanceResponse_ServerList: + s := proto.Size(x.ServerList) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +type InitialLoadBalanceResponse struct { + // This is an application layer redirect that indicates the client should use + // the specified server for load balancing. When this field is non-empty in + // the response, the client should open a separate connection to the + // load_balancer_delegate and call the BalanceLoad method. Its length should + // be less than 64 bytes. + LoadBalancerDelegate string `protobuf:"bytes,1,opt,name=load_balancer_delegate,json=loadBalancerDelegate" json:"load_balancer_delegate,omitempty"` + // This interval defines how often the client should send the client stats + // to the load balancer. Stats should only be reported when the duration is + // positive. + ClientStatsReportInterval *Duration `protobuf:"bytes,2,opt,name=client_stats_report_interval,json=clientStatsReportInterval" json:"client_stats_report_interval,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *InitialLoadBalanceResponse) Reset() { *m = InitialLoadBalanceResponse{} } +func (m *InitialLoadBalanceResponse) String() string { return proto.CompactTextString(m) } +func (*InitialLoadBalanceResponse) ProtoMessage() {} +func (*InitialLoadBalanceResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b81c731f0e83edbd, []int{6} +} +func (m *InitialLoadBalanceResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_InitialLoadBalanceResponse.Unmarshal(m, b) +} +func (m *InitialLoadBalanceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_InitialLoadBalanceResponse.Marshal(b, m, deterministic) +} +func (dst *InitialLoadBalanceResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_InitialLoadBalanceResponse.Merge(dst, src) +} +func (m *InitialLoadBalanceResponse) XXX_Size() int { + return xxx_messageInfo_InitialLoadBalanceResponse.Size(m) +} +func (m *InitialLoadBalanceResponse) XXX_DiscardUnknown() { + xxx_messageInfo_InitialLoadBalanceResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_InitialLoadBalanceResponse proto.InternalMessageInfo + +func (m *InitialLoadBalanceResponse) GetLoadBalancerDelegate() string { + if m != nil { + return m.LoadBalancerDelegate + } + return "" +} + +func (m *InitialLoadBalanceResponse) GetClientStatsReportInterval() *Duration { + if m != nil { + return m.ClientStatsReportInterval + } + return nil +} + +type ServerList struct { + // Contains a list of servers selected by the load balancer. The list will + // be updated when server resolutions change or as needed to balance load + // across more servers. The client should consume the server list in order + // unless instructed otherwise via the client_config. + Servers []*Server `protobuf:"bytes,1,rep,name=servers" json:"servers,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ServerList) Reset() { *m = ServerList{} } +func (m *ServerList) String() string { return proto.CompactTextString(m) } +func (*ServerList) ProtoMessage() {} +func (*ServerList) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b81c731f0e83edbd, []int{7} +} +func (m *ServerList) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ServerList.Unmarshal(m, b) +} +func (m *ServerList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ServerList.Marshal(b, m, deterministic) +} +func (dst *ServerList) XXX_Merge(src proto.Message) { + xxx_messageInfo_ServerList.Merge(dst, src) +} +func (m *ServerList) XXX_Size() int { + return xxx_messageInfo_ServerList.Size(m) +} +func (m *ServerList) XXX_DiscardUnknown() { + xxx_messageInfo_ServerList.DiscardUnknown(m) +} + +var xxx_messageInfo_ServerList proto.InternalMessageInfo + +func (m *ServerList) GetServers() []*Server { + if m != nil { + return m.Servers + } + return nil +} + +// Contains server information. When none of the [drop_for_*] fields are true, +// use the other fields. When drop_for_rate_limiting is true, ignore all other +// fields. Use drop_for_load_balancing only when it is true and +// drop_for_rate_limiting is false. +type Server struct { + // A resolved address for the server, serialized in network-byte-order. It may + // either be an IPv4 or IPv6 address. + IpAddress []byte `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` + // A resolved port number for the server. + Port int32 `protobuf:"varint,2,opt,name=port" json:"port,omitempty"` + // An opaque but printable token given to the frontend for each pick. All + // frontend requests for that pick must include the token in its initial + // metadata. The token is used by the backend to verify the request and to + // allow the backend to report load to the gRPC LB system. + // + // Its length is variable but less than 50 bytes. + LoadBalanceToken string `protobuf:"bytes,3,opt,name=load_balance_token,json=loadBalanceToken" json:"load_balance_token,omitempty"` + // Indicates whether this particular request should be dropped by the client + // for rate limiting. + DropForRateLimiting bool `protobuf:"varint,4,opt,name=drop_for_rate_limiting,json=dropForRateLimiting" json:"drop_for_rate_limiting,omitempty"` + // Indicates whether this particular request should be dropped by the client + // for load balancing. + DropForLoadBalancing bool `protobuf:"varint,5,opt,name=drop_for_load_balancing,json=dropForLoadBalancing" json:"drop_for_load_balancing,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Server) Reset() { *m = Server{} } +func (m *Server) String() string { return proto.CompactTextString(m) } +func (*Server) ProtoMessage() {} +func (*Server) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b81c731f0e83edbd, []int{8} +} +func (m *Server) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Server.Unmarshal(m, b) +} +func (m *Server) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Server.Marshal(b, m, deterministic) +} +func (dst *Server) XXX_Merge(src proto.Message) { + xxx_messageInfo_Server.Merge(dst, src) +} +func (m *Server) XXX_Size() int { + return xxx_messageInfo_Server.Size(m) +} +func (m *Server) XXX_DiscardUnknown() { + xxx_messageInfo_Server.DiscardUnknown(m) +} + +var xxx_messageInfo_Server proto.InternalMessageInfo + +func (m *Server) GetIpAddress() []byte { + if m != nil { + return m.IpAddress + } + return nil +} + +func (m *Server) GetPort() int32 { + if m != nil { + return m.Port + } + return 0 +} + +func (m *Server) GetLoadBalanceToken() string { + if m != nil { + return m.LoadBalanceToken + } + return "" +} + +func (m *Server) GetDropForRateLimiting() bool { + if m != nil { + return m.DropForRateLimiting + } + return false +} + +func (m *Server) GetDropForLoadBalancing() bool { + if m != nil { + return m.DropForLoadBalancing + } + return false +} + +func init() { + proto.RegisterType((*Duration)(nil), "grpc.lb.v1.Duration") + proto.RegisterType((*Timestamp)(nil), "grpc.lb.v1.Timestamp") + proto.RegisterType((*LoadBalanceRequest)(nil), "grpc.lb.v1.LoadBalanceRequest") + proto.RegisterType((*InitialLoadBalanceRequest)(nil), "grpc.lb.v1.InitialLoadBalanceRequest") + proto.RegisterType((*ClientStats)(nil), "grpc.lb.v1.ClientStats") + proto.RegisterType((*LoadBalanceResponse)(nil), "grpc.lb.v1.LoadBalanceResponse") + proto.RegisterType((*InitialLoadBalanceResponse)(nil), "grpc.lb.v1.InitialLoadBalanceResponse") + proto.RegisterType((*ServerList)(nil), "grpc.lb.v1.ServerList") + proto.RegisterType((*Server)(nil), "grpc.lb.v1.Server") +} + +func init() { + proto.RegisterFile("grpc_lb_v1/messages/messages.proto", fileDescriptor_messages_b81c731f0e83edbd) +} + +var fileDescriptor_messages_b81c731f0e83edbd = []byte{ + // 731 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xdd, 0x4e, 0x1b, 0x39, + 0x14, 0x26, 0x9b, 0x00, 0xc9, 0x09, 0x5a, 0xb2, 0x26, 0x0b, 0x81, 0x05, 0x89, 0x1d, 0x69, 0xd9, + 0x68, 0xc5, 0x4e, 0x04, 0xd9, 0xbd, 0xe8, 0xcf, 0x45, 0x1b, 0x10, 0x0a, 0x2d, 0x17, 0x95, 0x43, + 0x55, 0xa9, 0x52, 0x65, 0x39, 0x19, 0x33, 0x58, 0x38, 0xf6, 0xd4, 0x76, 0x82, 0xfa, 0x08, 0x7d, + 0x94, 0x3e, 0x46, 0xd5, 0x67, 0xe8, 0xfb, 0x54, 0xe3, 0x99, 0xc9, 0x0c, 0x10, 0x40, 0xbd, 0x89, + 0xec, 0xe3, 0xef, 0x7c, 0xdf, 0xf1, 0x89, 0xbf, 0x33, 0xe0, 0x85, 0x3a, 0x1a, 0x11, 0x31, 0x24, + 0xd3, 0x83, 0xce, 0x98, 0x19, 0x43, 0x43, 0x66, 0x66, 0x0b, 0x3f, 0xd2, 0xca, 0x2a, 0x04, 0x31, + 0xc6, 0x17, 0x43, 0x7f, 0x7a, 0xe0, 0x3d, 0x85, 0xea, 0xf1, 0x44, 0x53, 0xcb, 0x95, 0x44, 0x2d, + 0x58, 0x36, 0x6c, 0xa4, 0x64, 0x60, 0x5a, 0xa5, 0xdd, 0x52, 0xbb, 0x8c, 0xb3, 0x2d, 0x6a, 0xc2, + 0xa2, 0xa4, 0x52, 0x99, 0xd6, 0x2f, 0xbb, 0xa5, 0xf6, 0x22, 0x4e, 0x36, 0xde, 0x33, 0xa8, 0x9d, + 0xf3, 0x31, 0x33, 0x96, 0x8e, 0xa3, 0x9f, 0x4e, 0xfe, 0x5a, 0x02, 0x74, 0xa6, 0x68, 0xd0, 0xa3, + 0x82, 0xca, 0x11, 0xc3, 0xec, 0xe3, 0x84, 0x19, 0x8b, 0xde, 0xc0, 0x2a, 0x97, 0xdc, 0x72, 0x2a, + 0x88, 0x4e, 0x42, 0x8e, 0xae, 0x7e, 0xf8, 0x97, 0x9f, 0x57, 0xed, 0x9f, 0x26, 0x90, 0xbb, 0xf9, + 0xfd, 0x05, 0xfc, 0x6b, 0x9a, 0x9f, 0x31, 0x3e, 0x87, 0x95, 0x91, 0xe0, 0x4c, 0x5a, 0x62, 0x2c, + 0xb5, 0x49, 0x15, 0xf5, 0xc3, 0x8d, 0x22, 0xdd, 0x91, 0x3b, 0x1f, 0xc4, 0xc7, 0xfd, 0x05, 0x5c, + 0x1f, 0xe5, 0xdb, 0xde, 0x1f, 0xb0, 0x29, 0x14, 0x0d, 0xc8, 0x30, 0x91, 0xc9, 0x8a, 0x22, 0xf6, + 0x53, 0xc4, 0xbc, 0x0e, 0x6c, 0xde, 0x5b, 0x09, 0x42, 0x50, 0x91, 0x74, 0xcc, 0x5c, 0xf9, 0x35, + 0xec, 0xd6, 0xde, 0xe7, 0x0a, 0xd4, 0x0b, 0x62, 0xa8, 0x0b, 0x35, 0x9b, 0x75, 0x30, 0xbd, 0xe7, + 0xef, 0xc5, 0xc2, 0x66, 0xed, 0xc5, 0x39, 0x0e, 0xfd, 0x03, 0xbf, 0xc9, 0xc9, 0x98, 0x8c, 0xa8, + 0x10, 0x26, 0xbe, 0x93, 0xb6, 0x2c, 0x70, 0xb7, 0x2a, 0xe3, 0x55, 0x39, 0x19, 0x1f, 0xc5, 0xf1, + 0x41, 0x12, 0x46, 0xfb, 0x80, 0x72, 0xec, 0x05, 0x97, 0xdc, 0x5c, 0xb2, 0xa0, 0x55, 0x76, 0xe0, + 0x46, 0x06, 0x3e, 0x49, 0xe3, 0x88, 0x80, 0x7f, 0x17, 0x4d, 0xae, 0xb9, 0xbd, 0x24, 0x81, 0x56, + 0x11, 0xb9, 0x50, 0x9a, 0x68, 0x6a, 0x19, 0x11, 0x7c, 0xcc, 0x2d, 0x97, 0x61, 0xab, 0xe2, 0x98, + 0xfe, 0xbe, 0xcd, 0xf4, 0x8e, 0xdb, 0xcb, 0x63, 0xad, 0xa2, 0x13, 0xa5, 0x31, 0xb5, 0xec, 0x2c, + 0x85, 0x23, 0x0a, 0x9d, 0x47, 0x05, 0x0a, 0xed, 0x8e, 0x15, 0x16, 0x9d, 0x42, 0xfb, 0x01, 0x85, + 0xbc, 0xf7, 0xb1, 0xc4, 0x07, 0xf8, 0xf7, 0x3e, 0x89, 0xf4, 0x19, 0x5c, 0x50, 0x2e, 0x58, 0x40, + 0xac, 0x22, 0x86, 0xc9, 0xa0, 0xb5, 0xe4, 0x04, 0xf6, 0xe6, 0x09, 0x24, 0x7f, 0xd5, 0x89, 0xc3, + 0x9f, 0xab, 0x01, 0x93, 0x01, 0xea, 0xc3, 0x9f, 0x73, 0xe8, 0xaf, 0xa4, 0xba, 0x96, 0x44, 0xb3, + 0x11, 0xe3, 0x53, 0x16, 0xb4, 0x96, 0x1d, 0xe5, 0xce, 0x6d, 0xca, 0xd7, 0x31, 0x0a, 0xa7, 0x20, + 0xef, 0x5b, 0x09, 0xd6, 0x6e, 0x3c, 0x1b, 0x13, 0x29, 0x69, 0x18, 0x1a, 0x40, 0x23, 0x77, 0x40, + 0x12, 0x4b, 0x9f, 0xc6, 0xde, 0x63, 0x16, 0x48, 0xd0, 0xfd, 0x05, 0xbc, 0x3a, 0xf3, 0x40, 0x4a, + 0xfa, 0x04, 0xea, 0x86, 0xe9, 0x29, 0xd3, 0x44, 0x70, 0x63, 0x53, 0x0f, 0xac, 0x17, 0xf9, 0x06, + 0xee, 0xf8, 0x8c, 0x3b, 0x0f, 0x81, 0x99, 0xed, 0x7a, 0xdb, 0xb0, 0x75, 0xcb, 0x01, 0x09, 0x67, + 0x62, 0x81, 0x2f, 0x25, 0xd8, 0xba, 0xbf, 0x14, 0xf4, 0x1f, 0xac, 0x17, 0x93, 0x35, 0x09, 0x98, + 0x60, 0x21, 0xb5, 0x99, 0x2d, 0x9a, 0x22, 0x4f, 0xd2, 0xc7, 0xe9, 0x19, 0x7a, 0x0b, 0xdb, 0x45, + 0xcb, 0x12, 0xcd, 0x22, 0xa5, 0x2d, 0xe1, 0xd2, 0x32, 0x3d, 0xa5, 0x22, 0x2d, 0xbf, 0x59, 0x2c, + 0x3f, 0x1b, 0x62, 0x78, 0xb3, 0xe0, 0x5e, 0xec, 0xf2, 0x4e, 0xd3, 0x34, 0xef, 0x05, 0x40, 0x7e, + 0x4b, 0xb4, 0x1f, 0x0f, 0xac, 0x78, 0x17, 0x0f, 0xac, 0x72, 0xbb, 0x7e, 0x88, 0xee, 0xb6, 0x03, + 0x67, 0x90, 0x57, 0x95, 0x6a, 0xb9, 0x51, 0xf1, 0xbe, 0x97, 0x60, 0x29, 0x39, 0x41, 0x3b, 0x00, + 0x3c, 0x22, 0x34, 0x08, 0x34, 0x33, 0xc9, 0xc8, 0x5b, 0xc1, 0x35, 0x1e, 0xbd, 0x4c, 0x02, 0xb1, + 0xfb, 0x63, 0xed, 0x74, 0xe6, 0xb9, 0x75, 0x6c, 0xc6, 0x1b, 0x9d, 0xb4, 0xea, 0x8a, 0x49, 0x67, + 0xc6, 0x1a, 0x6e, 0x14, 0x1a, 0x71, 0x1e, 0xc7, 0x51, 0x17, 0xd6, 0x1f, 0x30, 0x5d, 0x15, 0xaf, + 0x05, 0x73, 0x0c, 0xf6, 0x3f, 0x6c, 0x3c, 0x64, 0xa4, 0x2a, 0x6e, 0x06, 0x73, 0x4c, 0xd3, 0xeb, + 0xbe, 0x3f, 0x08, 0x95, 0x0a, 0x05, 0xf3, 0x43, 0x25, 0xa8, 0x0c, 0x7d, 0xa5, 0xc3, 0x4e, 0xdc, + 0x0d, 0xf7, 0x23, 0x86, 0x9d, 0x39, 0x5f, 0x95, 0xe1, 0x92, 0xfb, 0x9a, 0x74, 0x7f, 0x04, 0x00, + 0x00, 0xff, 0xff, 0x8e, 0xd0, 0x70, 0xb7, 0x73, 0x06, 0x00, 0x00, +} diff --git a/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.proto b/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.proto new file mode 100644 index 00000000000..42d99c109fe --- /dev/null +++ b/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.proto @@ -0,0 +1,155 @@ +// Copyright 2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package grpc.lb.v1; +option go_package = "google.golang.org/grpc/grpclb/grpc_lb_v1/messages"; + +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} + +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} + +message LoadBalanceRequest { + oneof load_balance_request_type { + // This message should be sent on the first request to the load balancer. + InitialLoadBalanceRequest initial_request = 1; + + // The client stats should be periodically reported to the load balancer + // based on the duration defined in the InitialLoadBalanceResponse. + ClientStats client_stats = 2; + } +} + +message InitialLoadBalanceRequest { + // Name of load balanced service (IE, balancer.service.com) + // length should be less than 256 bytes. + string name = 1; +} + +// Contains client level statistics that are useful to load balancing. Each +// count except the timestamp should be reset to zero after reporting the stats. +message ClientStats { + // The timestamp of generating the report. + Timestamp timestamp = 1; + + // The total number of RPCs that started. + int64 num_calls_started = 2; + + // The total number of RPCs that finished. + int64 num_calls_finished = 3; + + // The total number of RPCs that were dropped by the client because of rate + // limiting. + int64 num_calls_finished_with_drop_for_rate_limiting = 4; + + // The total number of RPCs that were dropped by the client because of load + // balancing. + int64 num_calls_finished_with_drop_for_load_balancing = 5; + + // The total number of RPCs that failed to reach a server except dropped RPCs. + int64 num_calls_finished_with_client_failed_to_send = 6; + + // The total number of RPCs that finished and are known to have been received + // by a server. + int64 num_calls_finished_known_received = 7; +} + +message LoadBalanceResponse { + oneof load_balance_response_type { + // This message should be sent on the first response to the client. + InitialLoadBalanceResponse initial_response = 1; + + // Contains the list of servers selected by the load balancer. The client + // should send requests to these servers in the specified order. + ServerList server_list = 2; + } +} + +message InitialLoadBalanceResponse { + // This is an application layer redirect that indicates the client should use + // the specified server for load balancing. When this field is non-empty in + // the response, the client should open a separate connection to the + // load_balancer_delegate and call the BalanceLoad method. Its length should + // be less than 64 bytes. + string load_balancer_delegate = 1; + + // This interval defines how often the client should send the client stats + // to the load balancer. Stats should only be reported when the duration is + // positive. + Duration client_stats_report_interval = 2; +} + +message ServerList { + // Contains a list of servers selected by the load balancer. The list will + // be updated when server resolutions change or as needed to balance load + // across more servers. The client should consume the server list in order + // unless instructed otherwise via the client_config. + repeated Server servers = 1; + + // Was google.protobuf.Duration expiration_interval. + reserved 3; +} + +// Contains server information. When none of the [drop_for_*] fields are true, +// use the other fields. When drop_for_rate_limiting is true, ignore all other +// fields. Use drop_for_load_balancing only when it is true and +// drop_for_rate_limiting is false. +message Server { + // A resolved address for the server, serialized in network-byte-order. It may + // either be an IPv4 or IPv6 address. + bytes ip_address = 1; + + // A resolved port number for the server. + int32 port = 2; + + // An opaque but printable token given to the frontend for each pick. All + // frontend requests for that pick must include the token in its initial + // metadata. The token is used by the backend to verify the request and to + // allow the backend to report load to the gRPC LB system. + // + // Its length is variable but less than 50 bytes. + string load_balance_token = 3; + + // Indicates whether this particular request should be dropped by the client + // for rate limiting. + bool drop_for_rate_limiting = 4; + + // Indicates whether this particular request should be dropped by the client + // for load balancing. + bool drop_for_load_balancing = 5; +} diff --git a/vendor/google.golang.org/grpc/keepalive/keepalive.go b/vendor/google.golang.org/grpc/keepalive/keepalive.go new file mode 100644 index 00000000000..f8adc7e6d4f --- /dev/null +++ b/vendor/google.golang.org/grpc/keepalive/keepalive.go @@ -0,0 +1,65 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package keepalive defines configurable parameters for point-to-point healthcheck. +package keepalive + +import ( + "time" +) + +// ClientParameters is used to set keepalive parameters on the client-side. +// These configure how the client will actively probe to notice when a connection is broken +// and send pings so intermediaries will be aware of the liveness of the connection. +// Make sure these parameters are set in coordination with the keepalive policy on the server, +// as incompatible settings can result in closing of connection. +type ClientParameters struct { + // After a duration of this time if the client doesn't see any activity it pings the server to see if the transport is still alive. + Time time.Duration // The current default value is infinity. + // After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that + // the connection is closed. + Timeout time.Duration // The current default value is 20 seconds. + // If true, client runs keepalive checks even with no active RPCs. + PermitWithoutStream bool // false by default. +} + +// ServerParameters is used to set keepalive and max-age parameters on the server-side. +type ServerParameters struct { + // MaxConnectionIdle is a duration for the amount of time after which an idle connection would be closed by sending a GoAway. + // Idleness duration is defined since the most recent time the number of outstanding RPCs became zero or the connection establishment. + MaxConnectionIdle time.Duration // The current default value is infinity. + // MaxConnectionAge is a duration for the maximum amount of time a connection may exist before it will be closed by sending a GoAway. + // A random jitter of +/-10% will be added to MaxConnectionAge to spread out connection storms. + MaxConnectionAge time.Duration // The current default value is infinity. + // MaxConnectinoAgeGrace is an additive period after MaxConnectionAge after which the connection will be forcibly closed. + MaxConnectionAgeGrace time.Duration // The current default value is infinity. + // After a duration of this time if the server doesn't see any activity it pings the client to see if the transport is still alive. + Time time.Duration // The current default value is 2 hours. + // After having pinged for keepalive check, the server waits for a duration of Timeout and if no activity is seen even after that + // the connection is closed. + Timeout time.Duration // The current default value is 20 seconds. +} + +// EnforcementPolicy is used to set keepalive enforcement policy on the server-side. +// Server will close connection with a client that violates this policy. +type EnforcementPolicy struct { + // MinTime is the minimum amount of time a client should wait before sending a keepalive ping. + MinTime time.Duration // The current default value is 5 minutes. + // If true, server expects keepalive pings even when there are no active streams(RPCs). + PermitWithoutStream bool // false by default. +} diff --git a/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go b/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go new file mode 100644 index 00000000000..c1cabfc995f --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go @@ -0,0 +1,379 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package dns implements a dns resolver to be installed as the default resolver +// in grpc. +package dns + +import ( + "encoding/json" + "errors" + "fmt" + "math/rand" + "net" + "os" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/net/context" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" +) + +func init() { + resolver.Register(NewBuilder()) +} + +const ( + defaultPort = "443" + defaultFreq = time.Minute * 30 + golang = "GO" + // In DNS, service config is encoded in a TXT record via the mechanism + // described in RFC-1464 using the attribute name grpc_config. + txtAttribute = "grpc_config=" +) + +var ( + errMissingAddr = errors.New("missing address") + randomGen = rand.New(rand.NewSource(time.Now().UnixNano())) +) + +// NewBuilder creates a dnsBuilder which is used to factory DNS resolvers. +func NewBuilder() resolver.Builder { + return &dnsBuilder{freq: defaultFreq} +} + +type dnsBuilder struct { + // frequency of polling the DNS server. + freq time.Duration +} + +// Build creates and starts a DNS resolver that watches the name resolution of the target. +func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOption) (resolver.Resolver, error) { + host, port, err := parseTarget(target.Endpoint) + if err != nil { + return nil, err + } + + // IP address. + if net.ParseIP(host) != nil { + host, _ = formatIP(host) + addr := []resolver.Address{{Addr: host + ":" + port}} + i := &ipResolver{ + cc: cc, + ip: addr, + rn: make(chan struct{}, 1), + q: make(chan struct{}), + } + cc.NewAddress(addr) + go i.watcher() + return i, nil + } + + // DNS address (non-IP). + ctx, cancel := context.WithCancel(context.Background()) + d := &dnsResolver{ + freq: b.freq, + host: host, + port: port, + ctx: ctx, + cancel: cancel, + cc: cc, + t: time.NewTimer(0), + rn: make(chan struct{}, 1), + disableServiceConfig: opts.DisableServiceConfig, + } + + d.wg.Add(1) + go d.watcher() + return d, nil +} + +// Scheme returns the naming scheme of this resolver builder, which is "dns". +func (b *dnsBuilder) Scheme() string { + return "dns" +} + +// ipResolver watches for the name resolution update for an IP address. +type ipResolver struct { + cc resolver.ClientConn + ip []resolver.Address + // rn channel is used by ResolveNow() to force an immediate resolution of the target. + rn chan struct{} + q chan struct{} +} + +// ResolveNow resend the address it stores, no resolution is needed. +func (i *ipResolver) ResolveNow(opt resolver.ResolveNowOption) { + select { + case i.rn <- struct{}{}: + default: + } +} + +// Close closes the ipResolver. +func (i *ipResolver) Close() { + close(i.q) +} + +func (i *ipResolver) watcher() { + for { + select { + case <-i.rn: + i.cc.NewAddress(i.ip) + case <-i.q: + return + } + } +} + +// dnsResolver watches for the name resolution update for a non-IP target. +type dnsResolver struct { + freq time.Duration + host string + port string + ctx context.Context + cancel context.CancelFunc + cc resolver.ClientConn + // rn channel is used by ResolveNow() to force an immediate resolution of the target. + rn chan struct{} + t *time.Timer + // wg is used to enforce Close() to return after the watcher() goroutine has finished. + // Otherwise, data race will be possible. [Race Example] in dns_resolver_test we + // replace the real lookup functions with mocked ones to facilitate testing. + // If Close() doesn't wait for watcher() goroutine finishes, race detector sometimes + // will warns lookup (READ the lookup function pointers) inside watcher() goroutine + // has data race with replaceNetFunc (WRITE the lookup function pointers). + wg sync.WaitGroup + disableServiceConfig bool +} + +// ResolveNow invoke an immediate resolution of the target that this dnsResolver watches. +func (d *dnsResolver) ResolveNow(opt resolver.ResolveNowOption) { + select { + case d.rn <- struct{}{}: + default: + } +} + +// Close closes the dnsResolver. +func (d *dnsResolver) Close() { + d.cancel() + d.wg.Wait() + d.t.Stop() +} + +func (d *dnsResolver) watcher() { + defer d.wg.Done() + for { + select { + case <-d.ctx.Done(): + return + case <-d.t.C: + case <-d.rn: + } + result, sc := d.lookup() + // Next lookup should happen after an interval defined by d.freq. + d.t.Reset(d.freq) + d.cc.NewServiceConfig(sc) + d.cc.NewAddress(result) + } +} + +func (d *dnsResolver) lookupSRV() []resolver.Address { + var newAddrs []resolver.Address + _, srvs, err := lookupSRV(d.ctx, "grpclb", "tcp", d.host) + if err != nil { + grpclog.Infof("grpc: failed dns SRV record lookup due to %v.\n", err) + return nil + } + for _, s := range srvs { + lbAddrs, err := lookupHost(d.ctx, s.Target) + if err != nil { + grpclog.Infof("grpc: failed load balancer address dns lookup due to %v.\n", err) + continue + } + for _, a := range lbAddrs { + a, ok := formatIP(a) + if !ok { + grpclog.Errorf("grpc: failed IP parsing due to %v.\n", err) + continue + } + addr := a + ":" + strconv.Itoa(int(s.Port)) + newAddrs = append(newAddrs, resolver.Address{Addr: addr, Type: resolver.GRPCLB, ServerName: s.Target}) + } + } + return newAddrs +} + +func (d *dnsResolver) lookupTXT() string { + ss, err := lookupTXT(d.ctx, d.host) + if err != nil { + grpclog.Infof("grpc: failed dns TXT record lookup due to %v.\n", err) + return "" + } + var res string + for _, s := range ss { + res += s + } + + // TXT record must have "grpc_config=" attribute in order to be used as service config. + if !strings.HasPrefix(res, txtAttribute) { + grpclog.Warningf("grpc: TXT record %v missing %v attribute", res, txtAttribute) + return "" + } + return strings.TrimPrefix(res, txtAttribute) +} + +func (d *dnsResolver) lookupHost() []resolver.Address { + var newAddrs []resolver.Address + addrs, err := lookupHost(d.ctx, d.host) + if err != nil { + grpclog.Warningf("grpc: failed dns A record lookup due to %v.\n", err) + return nil + } + for _, a := range addrs { + a, ok := formatIP(a) + if !ok { + grpclog.Errorf("grpc: failed IP parsing due to %v.\n", err) + continue + } + addr := a + ":" + d.port + newAddrs = append(newAddrs, resolver.Address{Addr: addr}) + } + return newAddrs +} + +func (d *dnsResolver) lookup() ([]resolver.Address, string) { + newAddrs := d.lookupSRV() + // Support fallback to non-balancer address. + newAddrs = append(newAddrs, d.lookupHost()...) + if d.disableServiceConfig { + return newAddrs, "" + } + sc := d.lookupTXT() + return newAddrs, canaryingSC(sc) +} + +// formatIP returns ok = false if addr is not a valid textual representation of an IP address. +// If addr is an IPv4 address, return the addr and ok = true. +// If addr is an IPv6 address, return the addr enclosed in square brackets and ok = true. +func formatIP(addr string) (addrIP string, ok bool) { + ip := net.ParseIP(addr) + if ip == nil { + return "", false + } + if ip.To4() != nil { + return addr, true + } + return "[" + addr + "]", true +} + +// parseTarget takes the user input target string, returns formatted host and port info. +// If target doesn't specify a port, set the port to be the defaultPort. +// If target is in IPv6 format and host-name is enclosed in sqarue brackets, brackets +// are strippd when setting the host. +// examples: +// target: "www.google.com" returns host: "www.google.com", port: "443" +// target: "ipv4-host:80" returns host: "ipv4-host", port: "80" +// target: "[ipv6-host]" returns host: "ipv6-host", port: "443" +// target: ":80" returns host: "localhost", port: "80" +// target: ":" returns host: "localhost", port: "443" +func parseTarget(target string) (host, port string, err error) { + if target == "" { + return "", "", errMissingAddr + } + if ip := net.ParseIP(target); ip != nil { + // target is an IPv4 or IPv6(without brackets) address + return target, defaultPort, nil + } + if host, port, err = net.SplitHostPort(target); err == nil { + // target has port, i.e ipv4-host:port, [ipv6-host]:port, host-name:port + if host == "" { + // Keep consistent with net.Dial(): If the host is empty, as in ":80", the local system is assumed. + host = "localhost" + } + if port == "" { + // If the port field is empty(target ends with colon), e.g. "[::1]:", defaultPort is used. + port = defaultPort + } + return host, port, nil + } + if host, port, err = net.SplitHostPort(target + ":" + defaultPort); err == nil { + // target doesn't have port + return host, port, nil + } + return "", "", fmt.Errorf("invalid target address %v, error info: %v", target, err) +} + +type rawChoice struct { + ClientLanguage *[]string `json:"clientLanguage,omitempty"` + Percentage *int `json:"percentage,omitempty"` + ClientHostName *[]string `json:"clientHostName,omitempty"` + ServiceConfig *json.RawMessage `json:"serviceConfig,omitempty"` +} + +func containsString(a *[]string, b string) bool { + if a == nil { + return true + } + for _, c := range *a { + if c == b { + return true + } + } + return false +} + +func chosenByPercentage(a *int) bool { + if a == nil { + return true + } + return randomGen.Intn(100)+1 <= *a +} + +func canaryingSC(js string) string { + if js == "" { + return "" + } + var rcs []rawChoice + err := json.Unmarshal([]byte(js), &rcs) + if err != nil { + grpclog.Warningf("grpc: failed to parse service config json string due to %v.\n", err) + return "" + } + cliHostname, err := os.Hostname() + if err != nil { + grpclog.Warningf("grpc: failed to get client hostname due to %v.\n", err) + return "" + } + var sc string + for _, c := range rcs { + if !containsString(c.ClientLanguage, golang) || + !chosenByPercentage(c.Percentage) || + !containsString(c.ClientHostName, cliHostname) || + c.ServiceConfig == nil { + continue + } + sc = string(*c.ServiceConfig) + break + } + return sc +} diff --git a/vendor/google.golang.org/grpc/resolver/dns/go17.go b/vendor/google.golang.org/grpc/resolver/dns/go17.go new file mode 100644 index 00000000000..b466bc8f6d4 --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver/dns/go17.go @@ -0,0 +1,35 @@ +// +build go1.6, !go1.8 + +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package dns + +import ( + "net" + + "golang.org/x/net/context" +) + +var ( + lookupHost = func(ctx context.Context, host string) ([]string, error) { return net.LookupHost(host) } + lookupSRV = func(ctx context.Context, service, proto, name string) (string, []*net.SRV, error) { + return net.LookupSRV(service, proto, name) + } + lookupTXT = func(ctx context.Context, name string) ([]string, error) { return net.LookupTXT(name) } +) diff --git a/vendor/google.golang.org/grpc/resolver/dns/go18.go b/vendor/google.golang.org/grpc/resolver/dns/go18.go new file mode 100644 index 00000000000..fa34f14cad4 --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver/dns/go18.go @@ -0,0 +1,29 @@ +// +build go1.8 + +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package dns + +import "net" + +var ( + lookupHost = net.DefaultResolver.LookupHost + lookupSRV = net.DefaultResolver.LookupSRV + lookupTXT = net.DefaultResolver.LookupTXT +) diff --git a/vendor/google.golang.org/grpc/resolver/passthrough/passthrough.go b/vendor/google.golang.org/grpc/resolver/passthrough/passthrough.go new file mode 100644 index 00000000000..b76010d74d1 --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver/passthrough/passthrough.go @@ -0,0 +1,57 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package passthrough implements a pass-through resolver. It sends the target +// name without scheme back to gRPC as resolved address. +package passthrough + +import "google.golang.org/grpc/resolver" + +const scheme = "passthrough" + +type passthroughBuilder struct{} + +func (*passthroughBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOption) (resolver.Resolver, error) { + r := &passthroughResolver{ + target: target, + cc: cc, + } + r.start() + return r, nil +} + +func (*passthroughBuilder) Scheme() string { + return scheme +} + +type passthroughResolver struct { + target resolver.Target + cc resolver.ClientConn +} + +func (r *passthroughResolver) start() { + r.cc.NewAddress([]resolver.Address{{Addr: r.target.Endpoint}}) +} + +func (*passthroughResolver) ResolveNow(o resolver.ResolveNowOption) {} + +func (*passthroughResolver) Close() {} + +func init() { + resolver.Register(&passthroughBuilder{}) +} diff --git a/vendor/google.golang.org/grpc/resolver/resolver.go b/vendor/google.golang.org/grpc/resolver/resolver.go new file mode 100644 index 00000000000..506afac88ae --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver/resolver.go @@ -0,0 +1,154 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package resolver defines APIs for name resolution in gRPC. +// All APIs in this package are experimental. +package resolver + +var ( + // m is a map from scheme to resolver builder. + m = make(map[string]Builder) + // defaultScheme is the default scheme to use. + defaultScheme = "passthrough" +) + +// TODO(bar) install dns resolver in init(){}. + +// Register registers the resolver builder to the resolver map. b.Scheme will be +// used as the scheme registered with this builder. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Resolvers are +// registered with the same name, the one registered last will take effect. +func Register(b Builder) { + m[b.Scheme()] = b +} + +// Get returns the resolver builder registered with the given scheme. +// +// If no builder is register with the scheme, nil will be returned. +func Get(scheme string) Builder { + if b, ok := m[scheme]; ok { + return b + } + return nil +} + +// SetDefaultScheme sets the default scheme that will be used. +// The default default scheme is "passthrough". +func SetDefaultScheme(scheme string) { + defaultScheme = scheme +} + +// GetDefaultScheme gets the default scheme that will be used. +func GetDefaultScheme() string { + return defaultScheme +} + +// AddressType indicates the address type returned by name resolution. +type AddressType uint8 + +const ( + // Backend indicates the address is for a backend server. + Backend AddressType = iota + // GRPCLB indicates the address is for a grpclb load balancer. + GRPCLB +) + +// Address represents a server the client connects to. +// This is the EXPERIMENTAL API and may be changed or extended in the future. +type Address struct { + // Addr is the server address on which a connection will be established. + Addr string + // Type is the type of this address. + Type AddressType + // ServerName is the name of this address. + // + // e.g. if Type is GRPCLB, ServerName should be the name of the remote load + // balancer, not the name of the backend. + ServerName string + // Metadata is the information associated with Addr, which may be used + // to make load balancing decision. + Metadata interface{} +} + +// BuildOption includes additional information for the builder to create +// the resolver. +type BuildOption struct { + // DisableServiceConfig indicates whether resolver should fetch service config data. + DisableServiceConfig bool +} + +// ClientConn contains the callbacks for resolver to notify any updates +// to the gRPC ClientConn. +// +// This interface is to be implemented by gRPC. Users should not need a +// brand new implementation of this interface. For the situations like +// testing, the new implementation should embed this interface. This allows +// gRPC to add new methods to this interface. +type ClientConn interface { + // NewAddress is called by resolver to notify ClientConn a new list + // of resolved addresses. + // The address list should be the complete list of resolved addresses. + NewAddress(addresses []Address) + // NewServiceConfig is called by resolver to notify ClientConn a new + // service config. The service config should be provided as a json string. + NewServiceConfig(serviceConfig string) +} + +// Target represents a target for gRPC, as specified in: +// https://github.com/grpc/grpc/blob/master/doc/naming.md. +type Target struct { + Scheme string + Authority string + Endpoint string +} + +// Builder creates a resolver that will be used to watch name resolution updates. +type Builder interface { + // Build creates a new resolver for the given target. + // + // gRPC dial calls Build synchronously, and fails if the returned error is + // not nil. + Build(target Target, cc ClientConn, opts BuildOption) (Resolver, error) + // Scheme returns the scheme supported by this resolver. + // Scheme is defined at https://github.com/grpc/grpc/blob/master/doc/naming.md. + Scheme() string +} + +// ResolveNowOption includes additional information for ResolveNow. +type ResolveNowOption struct{} + +// Resolver watches for the updates on the specified target. +// Updates include address updates and service config updates. +type Resolver interface { + // ResolveNow will be called by gRPC to try to resolve the target name + // again. It's just a hint, resolver can ignore this if it's not necessary. + // + // It could be called multiple times concurrently. + ResolveNow(ResolveNowOption) + // Close closes the resolver. + Close() +} + +// UnregisterForTesting removes the resolver builder with the given scheme from the +// resolver map. +// This function is for testing only. +func UnregisterForTesting(scheme string) { + delete(m, scheme) +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 495806d8d07..6cca4a0746a 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -214,6 +214,30 @@ "revision": "b4deda0973fb4c70b50d226b1af49f3da59f5265", "revisionTime": "2018-04-30T18:52:41Z" }, + { + "checksumSHA1": "tkJPssYejSjuAwE2tdEnoEIj93Q=", + "path": "github.com/golang/protobuf/ptypes", + "revision": "70c277a8a150a8e069492e6600926300405c2884", + "revisionTime": "2018-05-18T21:57:37Z" + }, + { + "checksumSHA1": "3eqU9o+NMZSLM/coY5WDq7C1uKg=", + "path": "github.com/golang/protobuf/ptypes/any", + "revision": "70c277a8a150a8e069492e6600926300405c2884", + "revisionTime": "2018-05-18T21:57:37Z" + }, + { + "checksumSHA1": "ZIF0rnVzNLluFPqUebtJrVonMr4=", + "path": "github.com/golang/protobuf/ptypes/duration", + "revision": "70c277a8a150a8e069492e6600926300405c2884", + "revisionTime": "2018-05-18T21:57:37Z" + }, + { + "checksumSHA1": "1FJvuT0UllZaaS43kmPlx8oNiCs=", + "path": "github.com/golang/protobuf/ptypes/timestamp", + "revision": "70c277a8a150a8e069492e6600926300405c2884", + "revisionTime": "2018-05-18T21:57:37Z" + }, { "checksumSHA1": "g/V4qrXjUGG9B+e3hB+4NAYJ5Gs=", "path": "github.com/gorilla/context", @@ -591,6 +615,12 @@ "revision": "236b8f043b920452504e263bc21d354427127473", "revisionTime": "2017-02-06T03:21:01Z" }, + { + "checksumSHA1": "c3H2wB/3tGrw6VqCnlye+kSdoXU=", + "path": "golang.org/x/net/http/httpguts", + "revision": "8e0cdda24ed423affc8f35c241e5e9b16180338e", + "revisionTime": "2018-04-25T08:40:29Z" + }, { "checksumSHA1": "j4JQxmxQsZTYj/fbW/7q7g9W50s=", "path": "golang.org/x/net/http2", @@ -751,12 +781,30 @@ "revision": "dafb3384ad25363d928a9e97ce4ad3a2f0667e34", "revisionTime": "2017-02-07T18:07:33Z" }, + { + "checksumSHA1": "faFDXp++cLjLBlvsr+izZ+go1WU=", + "path": "golang.org/x/text/secure/bidirule", + "revision": "506f9d5c962f284575e88337e7d9296d27e729d3", + "revisionTime": "2017-01-31T20:38:18Z" + }, { "checksumSHA1": "ziMb9+ANGRJSSIuxYdRbA+cDRBQ=", "path": "golang.org/x/text/transform", "revision": "dafb3384ad25363d928a9e97ce4ad3a2f0667e34", "revisionTime": "2017-02-07T18:07:33Z" }, + { + "checksumSHA1": "VoN1N+QSzZGXJuTX2mVH7fAUXNM=", + "path": "golang.org/x/text/unicode/bidi", + "revision": "506f9d5c962f284575e88337e7d9296d27e729d3", + "revisionTime": "2017-01-31T20:38:18Z" + }, + { + "checksumSHA1": "Vircurgvsnt4k26havmxPM67PUA=", + "path": "golang.org/x/text/unicode/norm", + "revision": "506f9d5c962f284575e88337e7d9296d27e729d3", + "revisionTime": "2017-01-31T20:38:18Z" + }, { "checksumSHA1": "eFQDEix/mGnhwnFu/Hq63zMfrX8=", "path": "golang.org/x/time/rate", @@ -787,6 +835,12 @@ "revision": "1dce95f761bfeb6fe2aa3c4e9ffdb261b69b3fc4", "revisionTime": "2017-03-01T11:38:50Z" }, + { + "checksumSHA1": "2iGEpqdA3KpuCNSWW6KPCIhpXl4=", + "path": "google.golang.org/genproto/googleapis/rpc/status", + "revision": "11a468237815f3a3ddf9f7c6e8b6b3b382a24d15", + "revisionTime": "2018-05-18T17:53:38Z" + }, { "checksumSHA1": "BSqEul8NC7Q0aeIkrndbP0ruROw=", "path": "google.golang.org/grpc", @@ -801,6 +855,18 @@ "revision": "18957c5fcde0c3037144153d3db03756542007e5", "revisionTime": "2018-05-04T17:55:21Z" }, + { + "checksumSHA1": "lw+L836hLeH8+//le+C+ycddCCU=", + "path": "google.golang.org/grpc/balancer/base", + "revision": "c16e07b13065171d4025e2646d041a1c864bde46", + "revisionTime": "2018-05-22T00:10:29Z" + }, + { + "checksumSHA1": "DJ1AtOk4Pu7bqtUMob95Hw8HPNw=", + "path": "google.golang.org/grpc/balancer/roundrobin", + "revision": "c16e07b13065171d4025e2646d041a1c864bde46", + "revisionTime": "2018-05-22T00:10:29Z" + }, { "checksumSHA1": "DmKj+1hxJ/0T+e2kwF1EgUaVcQg=", "path": "google.golang.org/grpc/channelz", @@ -815,6 +881,12 @@ "version": "v1.0.5", "versionExact": "v1.0.5" }, + { + "checksumSHA1": "XH2WYcDNwVO47zYShREJjcYXm0Y=", + "path": "google.golang.org/grpc/connectivity", + "revision": "c16e07b13065171d4025e2646d041a1c864bde46", + "revisionTime": "2018-05-22T00:10:29Z" + }, { "checksumSHA1": "KthiDKNPHMeIu967enqtE4NaZzI=", "path": "google.golang.org/grpc/credentials", @@ -835,6 +907,12 @@ "revision": "18957c5fcde0c3037144153d3db03756542007e5", "revisionTime": "2018-05-04T17:55:21Z" }, + { + "checksumSHA1": "n+8rAQxWcf9LPJat2UHq2uVzH20=", + "path": "google.golang.org/grpc/grpclb/grpc_lb_v1/messages", + "revision": "c16e07b13065171d4025e2646d041a1c864bde46", + "revisionTime": "2018-05-22T00:10:29Z" + }, { "checksumSHA1": "ZPPSFisPDz2ANO4FBZIft+fRxyk=", "path": "google.golang.org/grpc/grpclog", @@ -851,6 +929,12 @@ "version": "v1.0.5", "versionExact": "v1.0.5" }, + { + "checksumSHA1": "hcuHgKp8W0wIzoCnNfKI8NUss5o=", + "path": "google.golang.org/grpc/keepalive", + "revision": "c16e07b13065171d4025e2646d041a1c864bde46", + "revisionTime": "2018-05-22T00:10:29Z" + }, { "checksumSHA1": "OjIAi5AzqlQ7kLtdAyjvdgMf6hc=", "path": "google.golang.org/grpc/metadata", @@ -875,6 +959,24 @@ "version": "v1.0.5", "versionExact": "v1.0.5" }, + { + "checksumSHA1": "QOKwFz4Zdfxfjs8czgCCtzM5bk4=", + "path": "google.golang.org/grpc/resolver", + "revision": "c16e07b13065171d4025e2646d041a1c864bde46", + "revisionTime": "2018-05-22T00:10:29Z" + }, + { + "checksumSHA1": "reN7P4MtzZYidpSFocLZJD70EGo=", + "path": "google.golang.org/grpc/resolver/dns", + "revision": "c16e07b13065171d4025e2646d041a1c864bde46", + "revisionTime": "2018-05-22T00:10:29Z" + }, + { + "checksumSHA1": "zs9M4xE8Lyg4wvuYvR00XoBxmuw=", + "path": "google.golang.org/grpc/resolver/passthrough", + "revision": "c16e07b13065171d4025e2646d041a1c864bde46", + "revisionTime": "2018-05-22T00:10:29Z" + }, { "checksumSHA1": "YclPgme2gT3S0hTkHVdE1zAxJdo=", "path": "google.golang.org/grpc/stats", From ca49c01df3cb22fff74dfa1a17bdb12c4adeb324 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 22 May 2018 20:07:48 +0300 Subject: [PATCH 15/54] Fix certificate pinning though proxy (#1723) When request routed through the proxy `DialTLS` is not used, and only `VerifyPeerCertificate` is supported. The reason behind two separate checks is that `DialTLS` supports specifying public keys per hostname, and `VerifyPeerCertificate` only global ones, e.g. `*`, and I want to retain "per" host functionality since we already offer it. Added proxy mock, and additional tests for pinning though proxy and proxy functionality itself. Fixes https://github.com/TykTechnologies/tyk/issues/1717 Travis updated to run tests on Go 1.10 --- .travis.yml | 3 + cert.go | 33 +++++++ cert_go1.10_test.go | 212 ++++++++++++++++++++++++++++++++++++++++++++ cert_test.go | 149 ------------------------------- helpers_test.go | 95 ++++++++++++++++++++ reverse_proxy.go | 8 +- 6 files changed, 350 insertions(+), 150 deletions(-) create mode 100644 cert_go1.10_test.go diff --git a/.travis.yml b/.travis.yml index 215c6906fd9..58339af6a67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,9 @@ matrix: include: - go: 1.9.x env: LATEST_GO=true # run linters and report coverage + - go: 1.10.x + env: LATEST_GO=true # run linters and report coverage + services: - redis-server diff --git a/cert.go b/cert.go index 1cdec52bd3d..603ecdcead9 100644 --- a/cert.go +++ b/cert.go @@ -95,6 +95,39 @@ func getUpstreamCertificate(host string, spec *APISpec) (cert *tls.Certificate) return certs[0] } +func verifyPeerCertificatePinnedCheck(spec *APISpec, tlsConfig *tls.Config) func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + if (spec == nil || len(spec.PinnedPublicKeys) == 0) && len(config.Global().Security.PinnedPublicKeys) == 0 { + return nil + } + + tlsConfig.InsecureSkipVerify = true + + whitelist := getPinnedPublicKeys("*", spec) + if len(whitelist) == 0 { + return nil + } + + return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + for _, rawCert := range rawCerts { + cert, _ := x509.ParseCertificate(rawCert) + pub, err := x509.MarshalPKIXPublicKey(cert.PublicKey) + if err != nil { + continue + } + + fingerprint := certs.HexSHA256(pub) + + for _, w := range whitelist { + if w == fingerprint { + return nil + } + } + } + + return errors.New("Certificate public key pinning error. Public keys do not match.") + } +} + func dialTLSPinnedCheck(spec *APISpec, tc *tls.Config) func(network, addr string) (net.Conn, error) { if (spec == nil || len(spec.PinnedPublicKeys) == 0) && len(config.Global().Security.PinnedPublicKeys) == 0 { return nil diff --git a/cert_go1.10_test.go b/cert_go1.10_test.go new file mode 100644 index 00000000000..5d2634637a1 --- /dev/null +++ b/cert_go1.10_test.go @@ -0,0 +1,212 @@ +// +build go1.10 + +package main + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "net/http" + "net/http/httptest" + "testing" + + "github.com/TykTechnologies/tyk/certs" + "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/test" +) + +func TestPublicKeyPinning(t *testing.T) { + _, _, _, serverCert := genServerCertificate() + x509Cert, _ := x509.ParseCertificate(serverCert.Certificate[0]) + pubDer, _ := x509.MarshalPKIXPublicKey(x509Cert.PublicKey) + pubPem := pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: pubDer}) + pubID, _ := CertificateManager.Add(pubPem, "") + defer CertificateManager.Delete(pubID) + + if pubID != certs.HexSHA256(pubDer) { + t.Error("Certmanager returned wrong pub key fingerprint:", certs.HexSHA256(pubDer), pubID) + } + + upstream := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + })) + upstream.TLS = &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{serverCert}, + } + + upstream.StartTLS() + defer upstream.Close() + + t.Run("Pub key match", func(t *testing.T) { + globalConf := config.Global() + // For host using pinning, it should ignore standard verification in all cases, e.g setting variable below does nothing + globalConf.ProxySSLInsecureSkipVerify = false + config.SetGlobal(globalConf) + defer resetTestConfig() + + ts := newTykTestServer() + defer ts.Close() + + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.PinnedPublicKeys = map[string]string{"127.0.0.1": pubID} + spec.Proxy.TargetURL = upstream.URL + }) + + ts.Run(t, test.TestCase{Code: 200}) + }) + + t.Run("Pub key not match", func(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.PinnedPublicKeys = map[string]string{"127.0.0.1": "wrong"} + spec.Proxy.TargetURL = upstream.URL + }) + + ts.Run(t, test.TestCase{Code: 500}) + }) + + t.Run("Global setting", func(t *testing.T) { + globalConf := config.Global() + globalConf.Security.PinnedPublicKeys = map[string]string{"127.0.0.1": "wrong"} + config.SetGlobal(globalConf) + defer resetTestConfig() + + ts := newTykTestServer() + defer ts.Close() + + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.Proxy.TargetURL = upstream.URL + }) + + ts.Run(t, test.TestCase{Code: 500}) + }) + + t.Run("Though proxy", func(t *testing.T) { + _, _, _, proxyCert := genServerCertificate() + proxy := initProxy("https", &tls.Config{ + Certificates: []tls.Certificate{proxyCert}, + }) + + globalConf := config.Global() + globalConf.ProxySSLInsecureSkipVerify = true + config.SetGlobal(globalConf) + defer resetTestConfig() + + defer proxy.Stop() + + ts := newTykTestServer() + defer ts.Close() + + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.Proxy.TargetURL = upstream.URL + spec.Proxy.Transport.ProxyURL = proxy.URL + spec.PinnedPublicKeys = map[string]string{"*": "wrong"} + }) + + ts.Run(t, test.TestCase{Code: 500}) + }) +} + +func TestProxyTransport(t *testing.T) { + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("test")) + })) + defer upstream.Close() + + globalConf := config.Global() + globalConf.ProxySSLInsecureSkipVerify = true + // force creating new transport on each reque + globalConf.MaxConnTime = -1 + config.SetGlobal(globalConf) + defer resetTestConfig() + + ts := newTykTestServer() + defer ts.Close() + + //matching ciphers + t.Run("Global: Cipher match", func(t *testing.T) { + globalConf.ProxySSLCipherSuites = []string{"TLS_RSA_WITH_AES_128_CBC_SHA"} + config.SetGlobal(globalConf) + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.Proxy.TargetURL = upstream.URL + }) + ts.Run(t, test.TestCase{Path: "/", Code: 200}) + }) + + t.Run("Global: Cipher not match", func(t *testing.T) { + globalConf.ProxySSLCipherSuites = []string{"TLS_RSA_WITH_RC4_128_SHA"} + config.SetGlobal(globalConf) + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.Proxy.TargetURL = upstream.URL + }) + ts.Run(t, test.TestCase{Path: "/", Code: 500}) + }) + + t.Run("API: Cipher override", func(t *testing.T) { + globalConf.ProxySSLCipherSuites = []string{"TLS_RSA_WITH_RC4_128_SHA"} + config.SetGlobal(globalConf) + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.Proxy.TargetURL = upstream.URL + spec.Proxy.Transport.SSLCipherSuites = []string{"TLS_RSA_WITH_AES_128_CBC_SHA"} + }) + + ts.Run(t, test.TestCase{Path: "/", Code: 200}) + }) + + t.Run("API: MinTLS not match", func(t *testing.T) { + globalConf.ProxySSLMinVersion = 772 + config.SetGlobal(globalConf) + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.Proxy.TargetURL = upstream.URL + spec.Proxy.Transport.SSLCipherSuites = []string{"TLS_RSA_WITH_AES_128_CBC_SHA"} + }) + + ts.Run(t, test.TestCase{Path: "/", Code: 500}) + }) + + t.Run("API: Invalid proxy", func(t *testing.T) { + globalConf.ProxySSLMinVersion = 771 + config.SetGlobal(globalConf) + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.Proxy.TargetURL = upstream.URL + spec.Proxy.Transport.SSLCipherSuites = []string{"TLS_RSA_WITH_AES_128_CBC_SHA"} + // Invalid proxy + spec.Proxy.Transport.ProxyURL = upstream.URL + }) + + ts.Run(t, test.TestCase{Path: "/", Code: 500}) + }) + + t.Run("API: Valid proxy", func(t *testing.T) { + _, _, _, proxyCert := genServerCertificate() + proxy := initProxy("https", &tls.Config{ + Certificates: []tls.Certificate{proxyCert}, + }) + defer proxy.Stop() + + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.Proxy.TargetURL = upstream.URL + spec.Proxy.Transport.SSLCipherSuites = []string{"TLS_RSA_WITH_AES_128_CBC_SHA"} + // Invalid proxy + spec.Proxy.Transport.ProxyURL = proxy.URL + }) + + client := getTLSClient(nil, nil) + client.Transport = &http.Transport{ + TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper), + } + ts.Run(t, test.TestCase{Path: "/", Code: 200, Client: client}) + }) +} diff --git a/cert_test.go b/cert_test.go index 9c333c4f132..9bb69e2f829 100644 --- a/cert_test.go +++ b/cert_test.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "crypto/rsa" "crypto/tls" - _ "crypto/tls" "crypto/x509" "encoding/pem" "fmt" @@ -472,78 +471,6 @@ func TestUpstreamMutualTLS(t *testing.T) { }) } -func TestPublicKeyPinning(t *testing.T) { - _, _, _, serverCert := genServerCertificate() - x509Cert, _ := x509.ParseCertificate(serverCert.Certificate[0]) - pubDer, _ := x509.MarshalPKIXPublicKey(x509Cert.PublicKey) - pubPem := pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: pubDer}) - pubID, _ := CertificateManager.Add(pubPem, "") - defer CertificateManager.Delete(pubID) - - if pubID != certs.HexSHA256(pubDer) { - t.Error("Certmanager returned wrong pub key fingerprint:", certs.HexSHA256(pubDer), pubID) - } - - upstream := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - })) - upstream.TLS = &tls.Config{ - InsecureSkipVerify: true, - Certificates: []tls.Certificate{serverCert}, - } - - upstream.StartTLS() - defer upstream.Close() - - t.Run("Pub key match", func(t *testing.T) { - globalConf := config.Global() - // For host using pinning, it should ignore standard verification in all cases, e.g setting variable below does nothing - globalConf.ProxySSLInsecureSkipVerify = false - config.SetGlobal(globalConf) - defer resetTestConfig() - - ts := newTykTestServer() - defer ts.Close() - - buildAndLoadAPI(func(spec *APISpec) { - spec.Proxy.ListenPath = "/" - spec.PinnedPublicKeys = map[string]string{"127.0.0.1": pubID} - spec.Proxy.TargetURL = upstream.URL - }) - - ts.Run(t, test.TestCase{Code: 200}) - }) - - t.Run("Pub key not match", func(t *testing.T) { - ts := newTykTestServer() - defer ts.Close() - - buildAndLoadAPI(func(spec *APISpec) { - spec.Proxy.ListenPath = "/" - spec.PinnedPublicKeys = map[string]string{"127.0.0.1": "wrong"} - spec.Proxy.TargetURL = upstream.URL - }) - - ts.Run(t, test.TestCase{Code: 500}) - }) - - t.Run("Global setting", func(t *testing.T) { - globalConf := config.Global() - globalConf.Security.PinnedPublicKeys = map[string]string{"127.0.0.1": "wrong"} - config.SetGlobal(globalConf) - defer resetTestConfig() - - ts := newTykTestServer() - defer ts.Close() - - buildAndLoadAPI(func(spec *APISpec) { - spec.Proxy.ListenPath = "/" - spec.Proxy.TargetURL = upstream.URL - }) - - ts.Run(t, test.TestCase{Code: 500}) - }) -} - func TestKeyWithCertificateTLS(t *testing.T) { _, _, combinedPEM, _ := genServerCertificate() serverCertID, _ := CertificateManager.Add(combinedPEM, "") @@ -683,79 +610,3 @@ func TestCipherSuites(t *testing.T) { ts.Run(t, test.TestCase{Client: client, Path: "/", ErrorMatch: "tls: handshake failure"}) }) } - -func TestProxyTransport(t *testing.T) { - upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("test")) - })) - defer upstream.Close() - - globalConf := config.Global() - globalConf.ProxySSLInsecureSkipVerify = true - // force creating new transport on each reque - globalConf.MaxConnTime = -1 - config.SetGlobal(globalConf) - defer resetTestConfig() - - ts := newTykTestServer() - defer ts.Close() - - //matching ciphers - t.Run("Global: Cipher match", func(t *testing.T) { - globalConf.ProxySSLCipherSuites = []string{"TLS_RSA_WITH_AES_128_CBC_SHA"} - config.SetGlobal(globalConf) - buildAndLoadAPI(func(spec *APISpec) { - spec.Proxy.ListenPath = "/" - spec.Proxy.TargetURL = upstream.URL - }) - ts.Run(t, test.TestCase{Path: "/", Code: 200}) - }) - - t.Run("Global: Cipher not match", func(t *testing.T) { - globalConf.ProxySSLCipherSuites = []string{"TLS_RSA_WITH_RC4_128_SHA"} - config.SetGlobal(globalConf) - buildAndLoadAPI(func(spec *APISpec) { - spec.Proxy.ListenPath = "/" - spec.Proxy.TargetURL = upstream.URL - }) - ts.Run(t, test.TestCase{Path: "/", Code: 500}) - }) - - t.Run("API: Cipher override", func(t *testing.T) { - globalConf.ProxySSLCipherSuites = []string{"TLS_RSA_WITH_RC4_128_SHA"} - config.SetGlobal(globalConf) - buildAndLoadAPI(func(spec *APISpec) { - spec.Proxy.ListenPath = "/" - spec.Proxy.TargetURL = upstream.URL - spec.Proxy.Transport.SSLCipherSuites = []string{"TLS_RSA_WITH_AES_128_CBC_SHA"} - }) - - ts.Run(t, test.TestCase{Path: "/", Code: 200}) - }) - - t.Run("API: MinTLS not match", func(t *testing.T) { - globalConf.ProxySSLMinVersion = 772 - config.SetGlobal(globalConf) - buildAndLoadAPI(func(spec *APISpec) { - spec.Proxy.ListenPath = "/" - spec.Proxy.TargetURL = upstream.URL - spec.Proxy.Transport.SSLCipherSuites = []string{"TLS_RSA_WITH_AES_128_CBC_SHA"} - }) - - ts.Run(t, test.TestCase{Path: "/", Code: 500}) - }) - - t.Run("API: Proxy", func(t *testing.T) { - globalConf.ProxySSLMinVersion = 771 - config.SetGlobal(globalConf) - buildAndLoadAPI(func(spec *APISpec) { - spec.Proxy.ListenPath = "/" - spec.Proxy.TargetURL = upstream.URL - spec.Proxy.Transport.SSLCipherSuites = []string{"TLS_RSA_WITH_AES_128_CBC_SHA"} - // Invalid proxy - spec.Proxy.Transport.ProxyURL = upstream.URL - }) - - ts.Run(t, test.TestCase{Path: "/", Code: 500}) - }) -} diff --git a/helpers_test.go b/helpers_test.go index 38782e21faf..732ce6f6b41 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -3,6 +3,7 @@ package main import ( "archive/zip" "context" + "crypto/tls" "encoding/json" "fmt" "io" @@ -630,3 +631,97 @@ func initDNSMock() { }).DialContext, } } + +// Taken from https://medium.com/@mlowicki/http-s-proxy-in-golang-in-less-than-100-lines-of-code-6a51c2f2c38c +type httpProxyHandler struct { + proto string + URL string + server *http.Server + listener net.Listener +} + +func (p *httpProxyHandler) handleTunneling(w http.ResponseWriter, r *http.Request) { + dest_conn, err := net.DialTimeout("tcp", r.Host, 10*time.Second) + if err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + hijacker, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "Hijacking not supported", http.StatusInternalServerError) + return + } + client_conn, _, err := hijacker.Hijack() + if err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + } + go p.transfer(dest_conn, client_conn) + go p.transfer(client_conn, dest_conn) +} + +func (p *httpProxyHandler) transfer(destination io.WriteCloser, source io.ReadCloser) { + defer destination.Close() + defer source.Close() + io.Copy(destination, source) +} +func (p *httpProxyHandler) handleHTTP(w http.ResponseWriter, req *http.Request) { + resp, err := http.DefaultTransport.RoundTrip(req) + if err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + defer resp.Body.Close() + p.copyHeader(w.Header(), resp.Header) + w.WriteHeader(resp.StatusCode) + io.Copy(w, resp.Body) +} + +func (p *httpProxyHandler) Stop() error { + return p.server.Close() +} + +func (p *httpProxyHandler) copyHeader(dst, src http.Header) { + for k, vv := range src { + for _, v := range vv { + dst.Add(k, v) + } + } +} + +func initProxy(proto string, tlsConfig *tls.Config) *httpProxyHandler { + proxy := &httpProxyHandler{proto: proto} + + proxy.server = &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodConnect { + proxy.handleTunneling(w, r) + } else { + proxy.handleHTTP(w, r) + } + }), + // Disable HTTP/2. + TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), + } + + var err error + + switch proto { + case "http": + proxy.listener, err = net.Listen("tcp", ":0") + case "https": + proxy.listener, err = tls.Listen("tcp", ":0", tlsConfig) + default: + log.Fatal("Unsupported proto scheme", proto) + } + + if err != nil { + log.Fatal(err) + } + + proxy.URL = proto + "://" + proxy.listener.Addr().String() + + go proxy.server.Serve(proxy.listener) + + return proxy +} diff --git a/reverse_proxy.go b/reverse_proxy.go index 65911d8829e..d710a9ea597 100644 --- a/reverse_proxy.go +++ b/reverse_proxy.go @@ -438,7 +438,13 @@ func httpTransport(timeOut int, rw http.ResponseWriter, req *http.Request, p *Re transport.TLSClientConfig.InsecureSkipVerify = true } - transport.DialTLS = dialTLSPinnedCheck(p.TykAPISpec, transport.TLSClientConfig) + // When request routed through the proxy `DialTLS` is not used, and only VerifyPeerCertificate is supported + // The reason behind two separate checks is that `DialTLS` supports specifying public keys per hostname, and `VerifyPeerCertificate` only global ones, e.g. `*` + if proxyURL, _ := transport.Proxy(req); proxyURL != nil { + transport.TLSClientConfig.VerifyPeerCertificate = verifyPeerCertificatePinnedCheck(p.TykAPISpec, transport.TLSClientConfig) + } else { + transport.DialTLS = dialTLSPinnedCheck(p.TykAPISpec, transport.TLSClientConfig) + } if p.TykAPISpec.GlobalConfig.ProxySSLMinVersion > 0 { transport.TLSClientConfig.MinVersion = p.TykAPISpec.GlobalConfig.ProxySSLMinVersion From 19b51a37f4a9b385d431ed7d4a72173d89061f74 Mon Sep 17 00:00:00 2001 From: Martin Buhr Date: Wed, 23 May 2018 17:09:59 +1200 Subject: [PATCH 16/54] Handles Dashboard API failures, fixes #1725 (#1726) Have added a test that simulates a running gateway with 1 definition, and then that same gateway pulling a malformed definition The gateway will default to what is already loaded. This introduces the potential for gateways to get out-of-step but ensures existing services stay loaded until there has been a successful reload. Perhaps a future improvement would be a signal to the dashboard notifying which gateways have successfully reloaded --- api_definition.go | 25 ++++++++------- api_definition_test.go | 70 ++++++++++++++++++++++++++++++++++++++++++ main.go | 9 ++++-- 3 files changed, 91 insertions(+), 13 deletions(-) diff --git a/api_definition.go b/api_definition.go index 01d28c0f89a..08b38c89847 100644 --- a/api_definition.go +++ b/api_definition.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/json" "errors" + "fmt" "io" "io/ioutil" "net/http" @@ -12,14 +13,13 @@ import ( "path/filepath" "regexp" "strings" + "sync" "sync/atomic" "text/template" "time" "github.com/rubyist/circuitbreaker" - "sync" - "github.com/TykTechnologies/gojsonschema" "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/config" @@ -225,7 +225,7 @@ func (a APIDefinitionLoader) MakeSpec(def *apidef.APIDefinition) *APISpec { } // FromDashboardService will connect and download ApiDefintions from a Tyk Dashboard instance. -func (a APIDefinitionLoader) FromDashboardService(endpoint, secret string) []*APISpec { +func (a APIDefinitionLoader) FromDashboardService(endpoint, secret string) ([]*APISpec, error) { // Get the definitions log.Debug("Calling: ", endpoint) newRequest, err := http.NewRequest("GET", endpoint, nil) @@ -244,16 +244,20 @@ func (a APIDefinitionLoader) FromDashboardService(endpoint, secret string) []*AP } resp, err := c.Do(newRequest) if err != nil { - log.Error("Request failed: ", err) - return nil + return nil, err } defer resp.Body.Close() if resp.StatusCode == http.StatusForbidden { body, _ := ioutil.ReadAll(resp.Body) - log.Error("Login failure, Response was: ", string(body)) reLogin() - return nil + return nil, fmt.Errorf("login failure, Response was: %v", string(body)) + } + + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + reLogin() + return nil, fmt.Errorf("dashboard API error, response was: %v", string(body)) } // Extract tagged APIs# @@ -264,9 +268,8 @@ func (a APIDefinitionLoader) FromDashboardService(endpoint, secret string) []*AP Nonce string } if err := json.NewDecoder(resp.Body).Decode(&list); err != nil { - log.Error("Failed to decode body: ", err) - log.Info("--> Retrying in 5s") - return nil + body, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("failed to decode body: %v body was: %v", err, string(body)) } // Extract tagged entries only @@ -308,7 +311,7 @@ func (a APIDefinitionLoader) FromDashboardService(endpoint, secret string) []*AP ServiceNonce = list.Nonce log.Debug("Loading APIS Finished: Nonce Set: ", ServiceNonce) - return specs + return specs, nil } // FromCloud will connect and download ApiDefintions from a Mongo DB instance. diff --git a/api_definition_test.go b/api_definition_test.go index 0ac08b39b6e..1f02973613e 100644 --- a/api_definition_test.go +++ b/api_definition_test.go @@ -511,3 +511,73 @@ func BenchmarkGetVersionFromRequest(b *testing.B) { } }) } + +func TestSyncAPISpecsDashboardJSONFailure(t *testing.T) { + // Mock Dashboard + callNum := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/system/apis" { + if callNum == 0 { + w.Write([]byte(`{"Status": "OK", "Nonce": "1", "Message": [{"api_definition": {}}]}`)) + } else { + w.Write([]byte(`{"Status": "OK", "Nonce": "1", "Message": "this is a string"`)) + } + + callNum += 1 + } else { + t.Fatal("Unknown dashboard API request", r) + } + })) + defer ts.Close() + + apisMu.Lock() + apisByID = make(map[string]*APISpec) + apisMu.Unlock() + + globalConf := config.Global() + globalConf.UseDBAppConfigs = true + globalConf.AllowInsecureConfigs = true + globalConf.DBAppConfOptions.ConnectionString = ts.URL + config.SetGlobal(globalConf) + + defer resetTestConfig() + + var wg sync.WaitGroup + wg.Add(1) + msg := redis.Message{Data: []byte(`{"Command": "ApiUpdated"}`)} + handled := func(got NotificationCommand) { + if want := NoticeApiUpdated; got != want { + t.Fatalf("want %q, got %q", want, got) + } + } + handleRedisEvent(msg, handled, wg.Done) + + // Since we already know that reload is queued + reloadTick <- time.Time{} + + // Wait for the reload to finish, then check it worked + wg.Wait() + apisMu.RLock() + if len(apisByID) != 1 { + t.Error("should return array with one spec", apisByID) + } + apisMu.RUnlock() + + // Second call + + var wg2 sync.WaitGroup + wg2.Add(1) + handleRedisEvent(msg, handled, wg2.Done) + + // Since we already know that reload is queued + reloadTick <- time.Time{} + + // Wait for the reload to finish, then check it worked + wg2.Wait() + apisMu.RLock() + if len(apisByID) != 1 { + t.Error("second call should return array with one spec", apisByID) + } + apisMu.RUnlock() + +} diff --git a/main.go b/main.go index c2050f396b2..72eba79052e 100644 --- a/main.go +++ b/main.go @@ -251,9 +251,14 @@ func syncAPISpecs() int { defer apisMu.Unlock() if config.Global().UseDBAppConfigs { - connStr := buildConnStr("/system/apis") - apiSpecs = loader.FromDashboardService(connStr, config.Global().NodeSecret) + tmpSpecs, err := loader.FromDashboardService(connStr, config.Global().NodeSecret) + if err != nil { + log.Error("failed to load API specs: ", err) + return 0 + } + + apiSpecs = tmpSpecs mainLog.Debug("Downloading API Configurations from Dashboard Service") } else if config.Global().SlaveOptions.UseRPC { From a0642861a056d5dee430405863df7ea6c962c7bc Mon Sep 17 00:00:00 2001 From: dencoded <33698537+dencoded@users.noreply.github.com> Date: Wed, 16 May 2018 01:55:10 -0400 Subject: [PATCH 17/54] added changes to do less i/o to Redis --- auth_manager.go | 37 +++++++++++++++++++++++++++++++------ middleware.go | 13 ++++++++++--- user/session.go | 13 +++++++------ 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/auth_manager.go b/auth_manager.go index 1b3fca137c7..60019facab6 100644 --- a/auth_manager.go +++ b/auth_manager.go @@ -44,8 +44,9 @@ type DefaultAuthorisationManager struct { } type DefaultSessionManager struct { - store storage.Handler - asyncWrites bool + store storage.Handler + asyncWrites bool + disableCacheSessionState bool } func (b *DefaultAuthorisationManager) Init(store storage.Handler) { @@ -85,6 +86,7 @@ func (b *DefaultAuthorisationManager) KeyExpired(newSession *user.SessionState) func (b *DefaultSessionManager) Init(store storage.Handler) { b.asyncWrites = config.Global().UseAsyncSessionWrite + b.disableCacheSessionState = config.Global().LocalSessionCache.DisableCacheSessionState b.store = store b.store.Connect() } @@ -120,10 +122,16 @@ func (b *DefaultSessionManager) UpdateSession(keyName string, session *user.Sess v, _ := json.Marshal(session) - // Keep the TTL + if hashed { + keyName = b.store.GetKeyPrefix() + keyName + } + + // async update and return if needed if b.asyncWrites { + b.renewSessionState(keyName, session) + if hashed { - go b.store.SetRawKey(b.store.GetKeyPrefix()+keyName, string(v), resetTTLTo) + go b.store.SetRawKey(keyName, string(v), resetTTLTo) return nil } @@ -131,11 +139,28 @@ func (b *DefaultSessionManager) UpdateSession(keyName string, session *user.Sess return nil } + // sync update + var err error if hashed { - return b.store.SetRawKey(b.store.GetKeyPrefix()+keyName, string(v), resetTTLTo) + err = b.store.SetRawKey(keyName, string(v), resetTTLTo) + } else { + err = b.store.SetKey(keyName, string(v), resetTTLTo) } - return b.store.SetKey(keyName, string(v), resetTTLTo) + if err == nil { + b.renewSessionState(keyName, session) + } + + return err +} + +func (b *DefaultSessionManager) renewSessionState(keyName string, session *user.SessionState) { + // we have new session state so renew first-seen hash to prevent + session.SetFirstSeenHash() + // delete it from session cache to have it re-populated next time + if !b.disableCacheSessionState { + SessionCache.Delete(keyName) + } } // RemoveSession removes session from storage diff --git a/middleware.go b/middleware.go index b1e984dd609..1e648c9efbe 100644 --- a/middleware.go +++ b/middleware.go @@ -269,10 +269,17 @@ func (t BaseMiddleware) ApplyPolicies(key string, session *user.SessionState) er tags[tag] = true } } - session.Tags = make([]string, 0, len(tags)) - for tag := range tags { - session.Tags = append(session.Tags, tag) + + // set tags + if len(tags) > 0 { + session.Tags = make([]string, 0, len(tags)) + for tag := range tags { + session.Tags = append(session.Tags, tag) + } + } else { + session.Tags = nil } + session.AccessRights = rights // Update the session in the session manager in case it gets called again return t.Spec.SessionManager.UpdateSession(key, session, session.Lifetime(t.Spec.SessionLifetime), false) diff --git a/user/session.go b/user/session.go index c8655a6c5d8..6cc0d300808 100644 --- a/user/session.go +++ b/user/session.go @@ -72,13 +72,11 @@ type SessionState struct { IdExtractorDeadline int64 `json:"id_extractor_deadline" msg:"id_extractor_deadline"` SessionLifetime int64 `bson:"session_lifetime" json:"session_lifetime"` - FirstSeenHash string `bson:"-" json:"-"` + firstSeenHash string } -var murmurHasher = murmur3.New32() - func (s *SessionState) SetFirstSeenHash() { - s.FirstSeenHash = s.Hash() + s.firstSeenHash = s.Hash() } func (s *SessionState) Hash() string { @@ -87,11 +85,14 @@ func (s *SessionState) Hash() string { log.Error("Error encoding session data: ", err) return "" } - return string(murmurHasher.Sum(encoded)[:]) + murmurHasher := murmur3.New32() + murmurHasher.Write(encoded) + murmurHasher.Sum32() + return string(murmurHasher.Sum(nil)[:]) } func (s *SessionState) HasChanged() bool { - return s.FirstSeenHash == "" || s.FirstSeenHash != s.Hash() + return s.firstSeenHash == "" || s.firstSeenHash != s.Hash() } func (s *SessionState) Lifetime(fallback int64) int64 { From 4dc47eec5ae082103825087e4c9744d145f68aed Mon Sep 17 00:00:00 2001 From: dencoded <33698537+dencoded@users.noreply.github.com> Date: Wed, 23 May 2018 13:43:19 -0400 Subject: [PATCH 18/54] benchmark session quota fix --- multiauth_test.go | 56 +++++++++++++++++++++++++++++---------------- mw_auth_key_test.go | 34 ++++++++++++++++++--------- 2 files changed, 59 insertions(+), 31 deletions(-) diff --git a/multiauth_test.go b/multiauth_test.go index 8858f7b874a..1dfb8dc4cb7 100644 --- a/multiauth_test.go +++ b/multiauth_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/justinas/alice" + "github.com/lonelycode/go-uuid/uuid" "github.com/TykTechnologies/tyk/user" ) @@ -34,7 +35,7 @@ const multiAuthDev = `{ } }` -func createMultiAuthKeyAuthSession() *user.SessionState { +func createMultiAuthKeyAuthSession(isBench bool) *user.SessionState { session := new(user.SessionState) session.Rate = 100.0 session.Allowance = session.Rate @@ -42,13 +43,18 @@ func createMultiAuthKeyAuthSession() *user.SessionState { session.Per = 1.0 session.QuotaRenewalRate = 300 // 5 minutes session.QuotaRenews = time.Now().Unix() - session.QuotaRemaining = 900 - session.QuotaMax = 10 + if isBench { + session.QuotaRemaining = 100000000 + session.QuotaMax = 100000000 + } else { + session.QuotaRemaining = 900 + session.QuotaMax = 10 + } session.AccessRights = map[string]user.AccessDefinition{"55": {APIName: "Tyk Multi Key Test", APIID: "55", Versions: []string{"default"}}} return session } -func createMultiBasicAuthSession() *user.SessionState { +func createMultiBasicAuthSession(isBench bool) *user.SessionState { session := new(user.SessionState) session.Rate = 8.0 session.Allowance = session.Rate @@ -81,24 +87,34 @@ func getMultiAuthStandardAndBasicAuthChain(spec *APISpec) http.Handler { return chain } -func testPrepareMultiSessionBA(t testing.TB) (*APISpec, *http.Request) { +func testPrepareMultiSessionBA(t testing.TB, isBench bool) (*APISpec, *http.Request) { spec := createSpecTest(t, multiAuthDev) // Create BA - baSession := createMultiBasicAuthSession() - username := "0987876" + baSession := createMultiBasicAuthSession(isBench) + username := "" + if isBench { + username = uuid.New() + } else { + username = "0987876" + } password := "TEST" // Basic auth sessions are stored as {org-id}{username}, so we need to append it here when we create the session. - spec.SessionManager.UpdateSession("default0987876", baSession, 60, false) + spec.SessionManager.UpdateSession("default"+username, baSession, 60, false) // Create key - session := createMultiAuthKeyAuthSession() - customToken := "84573485734587384888723487243" + session := createMultiAuthKeyAuthSession(isBench) + customToken := "" + if isBench { + customToken = uuid.New() + } else { + customToken = "84573485734587384888723487243" + } // AuthKey sessions are stored by {token} spec.SessionManager.UpdateSession(customToken, session, 60, false) - to_encode := strings.Join([]string{username, password}, ":") - encodedPass := base64.StdEncoding.EncodeToString([]byte(to_encode)) + toEncode := strings.Join([]string{username, password}, ":") + encodedPass := base64.StdEncoding.EncodeToString([]byte(toEncode)) req := testReq(t, "GET", "/", nil) req.Header.Set("Authorization", fmt.Sprintf("Basic %s", encodedPass)) @@ -108,7 +124,7 @@ func testPrepareMultiSessionBA(t testing.TB) (*APISpec, *http.Request) { } func TestMultiSession_BA_Standard_OK(t *testing.T) { - spec, req := testPrepareMultiSessionBA(t) + spec, req := testPrepareMultiSessionBA(t, false) recorder := httptest.NewRecorder() chain := getMultiAuthStandardAndBasicAuthChain(spec) @@ -122,7 +138,7 @@ func TestMultiSession_BA_Standard_OK(t *testing.T) { func BenchmarkMultiSession_BA_Standard_OK(b *testing.B) { b.ReportAllocs() - spec, req := testPrepareMultiSessionBA(b) + spec, req := testPrepareMultiSessionBA(b, true) recorder := httptest.NewRecorder() chain := getMultiAuthStandardAndBasicAuthChain(spec) @@ -139,14 +155,14 @@ func TestMultiSession_BA_Standard_Identity(t *testing.T) { spec := createSpecTest(t, multiAuthDev) // Create BA - baSession := createMultiBasicAuthSession() + baSession := createMultiBasicAuthSession(false) username := "0987876" password := "TEST" // Basic auth sessions are stored as {org-id}{username}, so we need to append it here when we create the session. spec.SessionManager.UpdateSession("default0987876", baSession, 60, false) // Create key - session := createMultiAuthKeyAuthSession() + session := createMultiAuthKeyAuthSession(false) customToken := "84573485734587384888723487243" // AuthKey sessions are stored by {token} spec.SessionManager.UpdateSession(customToken, session, 60, false) @@ -176,14 +192,14 @@ func TestMultiSession_BA_Standard_FAILBA(t *testing.T) { spec := createSpecTest(t, multiAuthDev) // Create BA - baSession := createMultiBasicAuthSession() + baSession := createMultiBasicAuthSession(false) username := "0987876" password := "WRONG" // Basic auth sessions are stored as {org-id}{username}, so we need to append it here when we create the session. spec.SessionManager.UpdateSession("default0987876", baSession, 60, false) // Create key - session := createMultiAuthKeyAuthSession() + session := createMultiAuthKeyAuthSession(false) customToken := "84573485734587384888723487243" // AuthKey sessions are stored by {token} spec.SessionManager.UpdateSession(customToken, session, 60, false) @@ -208,14 +224,14 @@ func TestMultiSession_BA_Standard_FAILAuth(t *testing.T) { spec := createSpecTest(t, multiAuthDev) // Create BA - baSession := createMultiBasicAuthSession() + baSession := createMultiBasicAuthSession(false) username := "0987876" password := "TEST" // Basic auth sessions are stored as {org-id}{username}, so we need to append it here when we create the session. spec.SessionManager.UpdateSession("default0987876", baSession, 60, false) // Create key - session := createMultiAuthKeyAuthSession() + session := createMultiAuthKeyAuthSession(false) customToken := "84573485734587384888723487243" // AuthKey sessions are stored by {token} spec.SessionManager.UpdateSession(customToken, session, 60, false) diff --git a/mw_auth_key_test.go b/mw_auth_key_test.go index 2455576fb53..4a439f48286 100644 --- a/mw_auth_key_test.go +++ b/mw_auth_key_test.go @@ -10,10 +10,12 @@ import ( "github.com/justinas/alice" + "github.com/lonelycode/go-uuid/uuid" + "github.com/TykTechnologies/tyk/user" ) -func createAuthKeyAuthSession() *user.SessionState { +func createAuthKeyAuthSession(isBench bool) *user.SessionState { session := new(user.SessionState) // essentially non-throttled session.Rate = 100.0 @@ -22,8 +24,13 @@ func createAuthKeyAuthSession() *user.SessionState { session.Per = 1.0 session.QuotaRenewalRate = 300 // 5 minutes session.QuotaRenews = time.Now().Unix() - session.QuotaRemaining = 10 - session.QuotaMax = 10 + if isBench { + session.QuotaRemaining = 100000000 + session.QuotaMax = 100000000 + } else { + session.QuotaRemaining = 10 + session.QuotaMax = 10 + } session.AccessRights = map[string]user.AccessDefinition{"31": {APIName: "Tyk Auth Key Test", APIID: "31", Versions: []string{"default"}}} return session } @@ -45,17 +52,22 @@ func getAuthKeyChain(spec *APISpec) http.Handler { return chain } -func testPrepareAuthKeySession(tb testing.TB, apiDef string) (string, *APISpec) { +func testPrepareAuthKeySession(tb testing.TB, apiDef string, isBench bool) (string, *APISpec) { spec := createSpecTest(tb, apiDef) - session := createAuthKeyAuthSession() - customToken := "54321111" + session := createAuthKeyAuthSession(isBench) + customToken := "" + if isBench { + customToken = uuid.New() + } else { + customToken = "54321111" + } // AuthKey sessions are stored by {token} spec.SessionManager.UpdateSession(customToken, session, 60, false) return customToken, spec } func TestBearerTokenAuthKeySession(t *testing.T) { - customToken, spec := testPrepareAuthKeySession(t, authKeyDef) + customToken, spec := testPrepareAuthKeySession(t, authKeyDef, false) recorder := httptest.NewRecorder() req := testReq(t, "GET", "/auth_key_test/", nil) @@ -74,7 +86,7 @@ func TestBearerTokenAuthKeySession(t *testing.T) { func BenchmarkBearerTokenAuthKeySession(b *testing.B) { b.ReportAllocs() - customToken, spec := testPrepareAuthKeySession(b, authKeyDef) + customToken, spec := testPrepareAuthKeySession(b, authKeyDef, true) recorder := httptest.NewRecorder() req := testReq(b, "GET", "/auth_key_test/", nil) @@ -109,7 +121,7 @@ const authKeyDef = `{ }` func TestMultiAuthBackwardsCompatibleSession(t *testing.T) { - customToken, spec := testPrepareAuthKeySession(t, multiAuthBackwardsCompatible) + customToken, spec := testPrepareAuthKeySession(t, multiAuthBackwardsCompatible, false) recorder := httptest.NewRecorder() req := testReq(t, "GET", fmt.Sprintf("/auth_key_test/?token=%s", customToken), nil) @@ -126,7 +138,7 @@ func TestMultiAuthBackwardsCompatibleSession(t *testing.T) { func BenchmarkMultiAuthBackwardsCompatibleSession(b *testing.B) { b.ReportAllocs() - customToken, spec := testPrepareAuthKeySession(b, multiAuthBackwardsCompatible) + customToken, spec := testPrepareAuthKeySession(b, multiAuthBackwardsCompatible, true) recorder := httptest.NewRecorder() req := testReq(b, "GET", fmt.Sprintf("/auth_key_test/?token=%s", customToken), nil) @@ -163,7 +175,7 @@ const multiAuthBackwardsCompatible = `{ func TestMultiAuthSession(t *testing.T) { spec := createSpecTest(t, multiAuthDef) - session := createAuthKeyAuthSession() + session := createAuthKeyAuthSession(false) customToken := "54321111" // AuthKey sessions are stored by {token} spec.SessionManager.UpdateSession(customToken, session, 60, false) From 67ce5ae605b315df696995d2e31795ae883da2dd Mon Sep 17 00:00:00 2001 From: dencoded <33698537+dencoded@users.noreply.github.com> Date: Wed, 23 May 2018 17:10:16 -0400 Subject: [PATCH 19/54] one more benchmark fix, populate session mem-cache if enabled --- middleware.go | 8 +++-- mw_hmac_test.go | 35 ++++++++++++--------- mw_jwt_test.go | 53 ++++++++++++++++++++------------ mw_organization_activity_test.go | 12 ++++---- mw_virtual_endpoint.go | 1 + 5 files changed, 68 insertions(+), 41 deletions(-) diff --git a/middleware.go b/middleware.go index 1e648c9efbe..4d88b4625a2 100644 --- a/middleware.go +++ b/middleware.go @@ -323,7 +323,9 @@ func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string) (user.Ses if found { // If exists, assume it has been authorized and pass on // cache it - go SessionCache.Set(cacheKey, session, cache.DefaultExpiration) + if !t.Spec.GlobalConfig.LocalSessionCache.DisableCacheSessionState { + go SessionCache.Set(cacheKey, session, cache.DefaultExpiration) + } // Check for a policy, if there is a policy, pull it and overwrite the session values if err := t.ApplyPolicies(key, &session); err != nil { @@ -341,7 +343,9 @@ func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string) (user.Ses log.Info("Recreating session for key: ", key) // cache it - go SessionCache.Set(cacheKey, session, cache.DefaultExpiration) + if !t.Spec.GlobalConfig.LocalSessionCache.DisableCacheSessionState { + go SessionCache.Set(cacheKey, session, cache.DefaultExpiration) + } // Check for a policy, if there is a policy, pull it and overwrite the session values if err := t.ApplyPolicies(key, &session); err != nil { diff --git a/mw_hmac_test.go b/mw_hmac_test.go index c392743d297..27ad145cba0 100644 --- a/mw_hmac_test.go +++ b/mw_hmac_test.go @@ -14,11 +14,12 @@ import ( "testing" "time" + "github.com/justinas/alice" + "github.com/lonelycode/go-uuid/uuid" + "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/user" - - "github.com/justinas/alice" ) const hmacAuthDef = `{ @@ -97,7 +98,7 @@ func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { } } -func testPrepareHMACAuthSessionPass(tb testing.TB, eventWG *sync.WaitGroup, withHeader bool) (string, *APISpec, *http.Request) { +func testPrepareHMACAuthSessionPass(tb testing.TB, eventWG *sync.WaitGroup, withHeader bool, isBench bool) (string, *APISpec, *http.Request, string) { spec := createSpecTest(tb, hmacAuthDef) session := createHMACAuthSession() @@ -109,8 +110,14 @@ func testPrepareHMACAuthSessionPass(tb testing.TB, eventWG *sync.WaitGroup, with "AuthFailure": {&testAuthFailEventHandler{cb}}, } - // Basic auth sessions are stored as {org-id}{username}, so we need to append it here when we create the session. - spec.SessionManager.UpdateSession("9876", session, 60, false) + sessionKey := "" + if isBench { + sessionKey = uuid.New() + } else { + sessionKey = "9876" + } + + spec.SessionManager.UpdateSession(sessionKey, session, 60, false) req := testReq(tb, "GET", "/", nil) @@ -141,17 +148,17 @@ func testPrepareHMACAuthSessionPass(tb testing.TB, eventWG *sync.WaitGroup, with sigString := base64.StdEncoding.EncodeToString(h.Sum(nil)) encodedString := url.QueryEscape(sigString) - return encodedString, spec, req + return encodedString, spec, req, sessionKey } func TestHMACAuthSessionPass(t *testing.T) { // Should not receive an AuthFailure event var eventWG sync.WaitGroup eventWG.Add(1) - encodedString, spec, req := testPrepareHMACAuthSessionPass(t, &eventWG, false) + encodedString, spec, req, sessionKey := testPrepareHMACAuthSessionPass(t, &eventWG, false, false) recorder := httptest.NewRecorder() - req.Header.Set("Authorization", fmt.Sprintf("Signature keyId=\"9876\",algorithm=\"hmac-sha1\",signature=\"%s\"", encodedString)) + req.Header.Set("Authorization", fmt.Sprintf("Signature keyId=\"%s\",algorithm=\"hmac-sha1\",signature=\"%s\"", sessionKey, encodedString)) chain := getHMACAuthChain(spec) chain.ServeHTTP(recorder, req) @@ -171,10 +178,10 @@ func BenchmarkHMACAuthSessionPass(b *testing.B) { var eventWG sync.WaitGroup eventWG.Add(b.N) - encodedString, spec, req := testPrepareHMACAuthSessionPass(b, &eventWG, false) + encodedString, spec, req, sessionKey := testPrepareHMACAuthSessionPass(b, &eventWG, false, true) recorder := httptest.NewRecorder() - req.Header.Set("Authorization", fmt.Sprintf("Signature keyId=\"9876\",algorithm=\"hmac-sha1\",signature=\"%s\"", encodedString)) + req.Header.Set("Authorization", fmt.Sprintf("Signature keyId=\"%s\",algorithm=\"hmac-sha1\",signature=\"%s\"", sessionKey, encodedString)) chain := getHMACAuthChain(spec) @@ -398,10 +405,10 @@ func TestHMACAuthSessionPassWithHeaderField(t *testing.T) { // Should not receive an AuthFailure event var eventWG sync.WaitGroup eventWG.Add(1) - encodedString, spec, req := testPrepareHMACAuthSessionPass(t, &eventWG, true) + encodedString, spec, req, sessionKey := testPrepareHMACAuthSessionPass(t, &eventWG, true, false) recorder := httptest.NewRecorder() - req.Header.Set("Authorization", fmt.Sprintf("Signature keyId=\"9876\",algorithm=\"hmac-sha1\",headers=\"(request-target) date x-test-1 x-test-2\",signature=\"%s\"", encodedString)) + req.Header.Set("Authorization", fmt.Sprintf("Signature keyId=\"%s\",algorithm=\"hmac-sha1\",headers=\"(request-target) date x-test-1 x-test-2\",signature=\"%s\"", sessionKey, encodedString)) chain := getHMACAuthChain(spec) chain.ServeHTTP(recorder, req) @@ -421,10 +428,10 @@ func BenchmarkHMACAuthSessionPassWithHeaderField(b *testing.B) { var eventWG sync.WaitGroup eventWG.Add(b.N) - encodedString, spec, req := testPrepareHMACAuthSessionPass(b, &eventWG, true) + encodedString, spec, req, sessionKey := testPrepareHMACAuthSessionPass(b, &eventWG, true, true) recorder := httptest.NewRecorder() - req.Header.Set("Authorization", fmt.Sprintf("Signature keyId=\"9876\",algorithm=\"hmac-sha1\",headers=\"(request-target) date x-test-1 x-test-2\",signature=\"%s\"", encodedString)) + req.Header.Set("Authorization", fmt.Sprintf("Signature keyId=\"%s\",algorithm=\"hmac-sha1\",headers=\"(request-target) date x-test-1 x-test-2\",signature=\"%s\"", sessionKey, encodedString)) chain := getHMACAuthChain(spec) diff --git a/mw_jwt_test.go b/mw_jwt_test.go index 042e172d542..8f4acf173b1 100644 --- a/mw_jwt_test.go +++ b/mw_jwt_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/dgrijalva/jwt-go" + "github.com/lonelycode/go-uuid/uuid" "github.com/TykTechnologies/tyk/test" "github.com/TykTechnologies/tyk/user" @@ -88,7 +89,7 @@ func createJWTSessionWithRSAWithPolicy(policyID string) *user.SessionState { // JWTSessionHMAC -func prepareJWTSessionHMAC(tb testing.TB) string { +func prepareJWTSessionHMAC(tb testing.TB, isBench bool) string { spec := buildAndLoadAPI(func(spec *APISpec) { spec.UseKeylessAccess = false spec.JWTSigningMethod = "hmac" @@ -98,6 +99,9 @@ func prepareJWTSessionHMAC(tb testing.TB) string { session := createJWTSession() tokenKID := testKey(tb, "token") + if isBench { + tokenKID += "-" + uuid.New() + } spec.SessionManager.UpdateSession(tokenKID, session, 60, false) jwtToken := createJWKTokenHMAC(func(t *jwt.Token) { @@ -113,7 +117,7 @@ func TestJWTSessionHMAC(t *testing.T) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionHMAC(t) + jwtToken := prepareJWTSessionHMAC(t, false) authHeaders := map[string]string{"authorization": jwtToken} t.Run("Request with valid JWT signed with HMAC", func(t *testing.T) { @@ -129,7 +133,7 @@ func BenchmarkJWTSessionHMAC(b *testing.B) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionHMAC(b) + jwtToken := prepareJWTSessionHMAC(b, true) authHeaders := map[string]string{"authorization": jwtToken} for i := 0; i < b.N; i++ { @@ -140,7 +144,7 @@ func BenchmarkJWTSessionHMAC(b *testing.B) { } // JWTSessionRSA -func prepareJWTSessionRSA(tb testing.TB) (*APISpec, string) { +func prepareJWTSessionRSA(tb testing.TB, isBench bool) (*APISpec, string) { spec := buildAndLoadAPI(func(spec *APISpec) { spec.UseKeylessAccess = false spec.JWTSigningMethod = "rsa" @@ -150,6 +154,9 @@ func prepareJWTSessionRSA(tb testing.TB) (*APISpec, string) { session := createJWTSessionWithRSA() tokenKID := testKey(tb, "token") + if isBench { + tokenKID += "-" + uuid.New() + } spec.SessionManager.UpdateSession(tokenKID, session, 60, false) jwtToken := createJWKToken(func(t *jwt.Token) { @@ -165,7 +172,7 @@ func TestJWTSessionRSA(t *testing.T) { ts := newTykTestServer() defer ts.Close() - _, jwtToken := prepareJWTSessionRSA(t) + _, jwtToken := prepareJWTSessionRSA(t, false) authHeaders := map[string]string{"authorization": jwtToken} t.Run("Request with valid JWT", func(t *testing.T) { @@ -181,7 +188,7 @@ func BenchmarkJWTSessionRSA(b *testing.B) { ts := newTykTestServer() defer ts.Close() - _, jwtToken := prepareJWTSessionRSA(b) + _, jwtToken := prepareJWTSessionRSA(b, true) authHeaders := map[string]string{"authorization": jwtToken} for i := 0; i < b.N; i++ { @@ -195,7 +202,7 @@ func TestJWTSessionFailRSA_EmptyJWT(t *testing.T) { ts := newTykTestServer() defer ts.Close() - prepareJWTSessionRSA(t) + prepareJWTSessionRSA(t, false) authHeaders := map[string]string{"authorization": ""} t.Run("Request with empty authorization header", func(t *testing.T) { @@ -209,7 +216,7 @@ func TestJWTSessionFailRSA_NoAuthHeader(t *testing.T) { ts := newTykTestServer() defer ts.Close() - prepareJWTSessionRSA(t) + prepareJWTSessionRSA(t, false) authHeaders := map[string]string{} t.Run("Request without authorization header", func(t *testing.T) { @@ -223,7 +230,7 @@ func TestJWTSessionFailRSA_MalformedJWT(t *testing.T) { ts := newTykTestServer() defer ts.Close() - _, jwtToken := prepareJWTSessionRSA(t) + _, jwtToken := prepareJWTSessionRSA(t, false) authHeaders := map[string]string{"authorization": jwtToken + "ajhdkjhsdfkjashdkajshdkajhsdkajhsd"} t.Run("Request with malformed JWT", func(t *testing.T) { @@ -237,7 +244,7 @@ func TestJWTSessionFailRSA_MalformedJWT_NOTRACK(t *testing.T) { ts := newTykTestServer() defer ts.Close() - spec, jwtToken := prepareJWTSessionRSA(t) + spec, jwtToken := prepareJWTSessionRSA(t, false) spec.DoNotTrack = true authHeaders := map[string]string{"authorization": jwtToken + "ajhdkjhsdfkjashdkajshdkajhsdkajhsd"} @@ -252,7 +259,7 @@ func TestJWTSessionFailRSA_WrongJWT(t *testing.T) { ts := newTykTestServer() defer ts.Close() - prepareJWTSessionRSA(t) + prepareJWTSessionRSA(t, false) authHeaders := map[string]string{"authorization": "123"} t.Run("Request with invalid JWT", func(t *testing.T) { @@ -264,7 +271,7 @@ func TestJWTSessionFailRSA_WrongJWT(t *testing.T) { // TestJWTSessionRSABearer -func prepareJWTSessionRSABearer(tb testing.TB) string { +func prepareJWTSessionRSABearer(tb testing.TB, isBench bool) string { spec := buildAndLoadAPI(func(spec *APISpec) { spec.UseKeylessAccess = false spec.JWTSigningMethod = "rsa" @@ -274,6 +281,9 @@ func prepareJWTSessionRSABearer(tb testing.TB) string { session := createJWTSessionWithRSA() tokenKID := testKey(tb, "token") + if isBench { + tokenKID += "-" + uuid.New() + } spec.SessionManager.UpdateSession(tokenKID, session, 60, false) jwtToken := createJWKToken(func(t *jwt.Token) { @@ -289,7 +299,7 @@ func TestJWTSessionRSABearer(t *testing.T) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionRSABearer(t) + jwtToken := prepareJWTSessionRSABearer(t, false) authHeaders := map[string]string{"authorization": "Bearer " + jwtToken} t.Run("Request with valid Bearer", func(t *testing.T) { @@ -305,7 +315,7 @@ func BenchmarkJWTSessionRSABearer(b *testing.B) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionRSABearer(b) + jwtToken := prepareJWTSessionRSABearer(b, true) authHeaders := map[string]string{"authorization": "Bearer " + jwtToken} @@ -320,7 +330,7 @@ func TestJWTSessionRSABearerInvalid(t *testing.T) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionRSABearer(t) + jwtToken := prepareJWTSessionRSABearer(t, false) authHeaders := map[string]string{"authorization": "Bearer: " + jwtToken} // extra ":" t.Run("Request with invalid Bearer", func(t *testing.T) { @@ -332,7 +342,7 @@ func TestJWTSessionRSABearerInvalid(t *testing.T) { // JWTSessionRSAWithRawSourceOnWithClientID -func prepareJWTSessionRSAWithRawSourceOnWithClientID() string { +func prepareJWTSessionRSAWithRawSourceOnWithClientID(isBench bool) string { spec := buildAndLoadAPI(func(spec *APISpec) { spec.APIID = "777888" spec.OrgID = "default" @@ -356,7 +366,12 @@ func prepareJWTSessionRSAWithRawSourceOnWithClientID() string { } }) - tokenID := "1234567891010101" + tokenID := "" + if isBench { + tokenID = uuid.New() + } else { + tokenID = "1234567891010101" + } session := createJWTSessionWithRSAWithPolicy(policyID) spec.SessionManager.ResetQuota(tokenID, session) @@ -377,7 +392,7 @@ func TestJWTSessionRSAWithRawSourceOnWithClientID(t *testing.T) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionRSAWithRawSourceOnWithClientID() + jwtToken := prepareJWTSessionRSAWithRawSourceOnWithClientID(false) authHeaders := map[string]string{"authorization": jwtToken} t.Run("Initial request with no policy base field in JWT", func(t *testing.T) { @@ -393,7 +408,7 @@ func BenchmarkJWTSessionRSAWithRawSourceOnWithClientID(b *testing.B) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionRSAWithRawSourceOnWithClientID() + jwtToken := prepareJWTSessionRSAWithRawSourceOnWithClientID(true) authHeaders := map[string]string{"authorization": jwtToken} for i := 0; i < b.N; i++ { diff --git a/mw_organization_activity_test.go b/mw_organization_activity_test.go index bb25fd50383..e45fd30f558 100644 --- a/mw_organization_activity_test.go +++ b/mw_organization_activity_test.go @@ -97,9 +97,9 @@ func BenchmarkProcessRequestLiveQuotaLimit(b *testing.B) { b, ts, map[string]interface{}{ - "quota_max": 1000000, - "quota_remaining": 1000000, - "quota_renewal_rate": 3, + "quota_max": 100000000, + "quota_remaining": 100000000, + "quota_renewal_rate": 300, }, ) @@ -199,9 +199,9 @@ func BenchmarkProcessRequestOffThreadQuotaLimit(b *testing.B) { b, ts, map[string]interface{}{ - "quota_max": 1000000, - "quota_remaining": 1000000, - "quota_renewal_rate": 3, + "quota_max": 100000000, + "quota_remaining": 100000000, + "quota_renewal_rate": 300, }, ) diff --git a/mw_virtual_endpoint.go b/mw_virtual_endpoint.go index 86e3c212e8c..63d1f0b7d98 100644 --- a/mw_virtual_endpoint.go +++ b/mw_virtual_endpoint.go @@ -216,6 +216,7 @@ func (d *VirtualEndpoint) ServeHTTPForCache(w http.ResponseWriter, r *http.Reque if vmeta.UseSession { session.MetaData = mapStrsToIfaces(newResponseData.SessionMeta) d.Spec.SessionManager.UpdateSession(token, session, session.Lifetime(d.Spec.SessionLifetime), false) + ctxSetSession(r, session) } log.Debug("JSVM Virtual Endpoint execution took: (ns) ", time.Now().UnixNano()-t1) From 79feea1f25a272fbeae4902d61d7e4eabbb51440 Mon Sep 17 00:00:00 2001 From: Josh Blakeley <31618778+joshblakeley@users.noreply.github.com> Date: Thu, 24 May 2018 12:20:02 +0100 Subject: [PATCH 20/54] Expose url scheme to jsvm and add test (#1719) fixes #1516 --- mw_js_plugin.go | 4 ++-- mw_js_plugin_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/mw_js_plugin.go b/mw_js_plugin.go index d9e5196e50c..48272b5950f 100644 --- a/mw_js_plugin.go +++ b/mw_js_plugin.go @@ -45,7 +45,7 @@ type MiniRequestObject struct { IgnoreBody bool Method string RequestURI string - Proto string + Scheme string } type VMReturnObject struct { @@ -122,7 +122,7 @@ func (d *DynamicMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Reques DeleteParams: []string{}, Method: r.Method, RequestURI: r.RequestURI, - Proto: r.Proto, + Scheme: r.URL.Scheme, } requestAsJson, err := json.Marshal(requestData) diff --git a/mw_js_plugin_test.go b/mw_js_plugin_test.go index 9f2b7666046..bd20a8a6900 100644 --- a/mw_js_plugin_test.go +++ b/mw_js_plugin_test.go @@ -306,3 +306,44 @@ testJSVMCore.NewProcessRequest(function(request, session, config) { t.Fatalf("wanted header to be %q, got %q", want, got) } } +func TestJSVMRequestScheme(t *testing.T) { + dynMid := &DynamicMiddleware{ + BaseMiddleware: BaseMiddleware{ + Spec: &APISpec{APIDefinition: &apidef.APIDefinition{}}, + }, + MiddlewareClassName: "leakMid", + Pre: true, + } + req := httptest.NewRequest("GET", "/foo", nil) + req.URL.Scheme = "http" + jsvm := JSVM{} + jsvm.Init(nil) + + const js = ` +var leakMid = new TykJS.TykMiddleware.NewMiddleware({}) + +leakMid.NewProcessRequest(function(request, session) { + var test = request.Scheme += " appended" + + var responseObject = { + Body: test, + Code: 200 + } + return leakMid.ReturnData(responseObject, session.meta_data) +});` + if _, err := jsvm.VM.Run(js); err != nil { + t.Fatalf("failed to set up js plugin: %v", err) + } + dynMid.Spec.JSVM = jsvm + dynMid.ProcessRequest(nil, req, nil) + + bs, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Fatalf("failed to read final body: %v", err) + } + want := "http" + " appended" + if got := string(bs); want != got { + t.Fatalf("JS plugin broke non-UTF8 body %q into %q", + want, got) + } +} From 8b7d3feace4a8fc50896d5d9b974cc59b01506a6 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Mon, 21 May 2018 20:24:53 -0400 Subject: [PATCH 21/54] coprocess: replace proto with URL scheme --- coprocess.go | 2 +- .../coprocess_mini_request_object_pb2.py | 26 ++++---- .../ruby/coprocess_mini_request_object_pb.rb | 2 +- coprocess/coprocess_mini_request_object.pb.go | 62 +++++++++---------- .../proto/coprocess_mini_request_object.proto | 2 +- 5 files changed, 47 insertions(+), 47 deletions(-) diff --git a/coprocess.go b/coprocess.go index 0f2996a3fa5..3b78758d46a 100644 --- a/coprocess.go +++ b/coprocess.go @@ -101,7 +101,7 @@ func (c *CoProcessor) ObjectFromRequest(r *http.Request) *coprocess.Object { }, Method: r.Method, RequestUri: r.RequestURI, - Proto: r.Proto, + Scheme: r.URL.Scheme, } object := &coprocess.Object{ diff --git a/coprocess/bindings/python/coprocess_mini_request_object_pb2.py b/coprocess/bindings/python/coprocess_mini_request_object_pb2.py index d604f096c32..a287f3d27ea 100644 --- a/coprocess/bindings/python/coprocess_mini_request_object_pb2.py +++ b/coprocess/bindings/python/coprocess_mini_request_object_pb2.py @@ -20,7 +20,7 @@ name='coprocess_mini_request_object.proto', package='coprocess', syntax='proto3', - serialized_pb=_b('\n#coprocess_mini_request_object.proto\x12\tcoprocess\x1a coprocess_return_overrides.proto\"\x87\x06\n\x11MiniRequestObject\x12:\n\x07headers\x18\x01 \x03(\x0b\x32).coprocess.MiniRequestObject.HeadersEntry\x12\x41\n\x0bset_headers\x18\x02 \x03(\x0b\x32,.coprocess.MiniRequestObject.SetHeadersEntry\x12\x16\n\x0e\x64\x65lete_headers\x18\x03 \x03(\t\x12\x0c\n\x04\x62ody\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x38\n\x06params\x18\x06 \x03(\x0b\x32(.coprocess.MiniRequestObject.ParamsEntry\x12?\n\nadd_params\x18\x07 \x03(\x0b\x32+.coprocess.MiniRequestObject.AddParamsEntry\x12I\n\x0f\x65xtended_params\x18\x08 \x03(\x0b\x32\x30.coprocess.MiniRequestObject.ExtendedParamsEntry\x12\x15\n\rdelete_params\x18\t \x03(\t\x12\x34\n\x10return_overrides\x18\n \x01(\x0b\x32\x1a.coprocess.ReturnOverrides\x12\x0e\n\x06method\x18\x0b \x01(\t\x12\x13\n\x0brequest_uri\x18\x0c \x01(\t\x12\r\n\x05proto\x18\r \x01(\t\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fSetHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a-\n\x0bParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x30\n\x0e\x41\x64\x64ParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x35\n\x13\x45xtendedParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3') + serialized_pb=_b('\n#coprocess_mini_request_object.proto\x12\tcoprocess\x1a coprocess_return_overrides.proto\"\x88\x06\n\x11MiniRequestObject\x12:\n\x07headers\x18\x01 \x03(\x0b\x32).coprocess.MiniRequestObject.HeadersEntry\x12\x41\n\x0bset_headers\x18\x02 \x03(\x0b\x32,.coprocess.MiniRequestObject.SetHeadersEntry\x12\x16\n\x0e\x64\x65lete_headers\x18\x03 \x03(\t\x12\x0c\n\x04\x62ody\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x38\n\x06params\x18\x06 \x03(\x0b\x32(.coprocess.MiniRequestObject.ParamsEntry\x12?\n\nadd_params\x18\x07 \x03(\x0b\x32+.coprocess.MiniRequestObject.AddParamsEntry\x12I\n\x0f\x65xtended_params\x18\x08 \x03(\x0b\x32\x30.coprocess.MiniRequestObject.ExtendedParamsEntry\x12\x15\n\rdelete_params\x18\t \x03(\t\x12\x34\n\x10return_overrides\x18\n \x01(\x0b\x32\x1a.coprocess.ReturnOverrides\x12\x0e\n\x06method\x18\x0b \x01(\t\x12\x13\n\x0brequest_uri\x18\x0c \x01(\t\x12\x0e\n\x06scheme\x18\r \x01(\t\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fSetHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a-\n\x0bParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x30\n\x0e\x41\x64\x64ParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x35\n\x13\x45xtendedParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3') , dependencies=[coprocess__return__overrides__pb2.DESCRIPTOR,]) @@ -60,8 +60,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=611, - serialized_end=657, + serialized_start=612, + serialized_end=658, ) _MINIREQUESTOBJECT_SETHEADERSENTRY = _descriptor.Descriptor( @@ -97,8 +97,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=659, - serialized_end=708, + serialized_start=660, + serialized_end=709, ) _MINIREQUESTOBJECT_PARAMSENTRY = _descriptor.Descriptor( @@ -134,8 +134,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=710, - serialized_end=755, + serialized_start=711, + serialized_end=756, ) _MINIREQUESTOBJECT_ADDPARAMSENTRY = _descriptor.Descriptor( @@ -171,8 +171,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=757, - serialized_end=805, + serialized_start=758, + serialized_end=806, ) _MINIREQUESTOBJECT_EXTENDEDPARAMSENTRY = _descriptor.Descriptor( @@ -208,8 +208,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=807, - serialized_end=860, + serialized_start=808, + serialized_end=861, ) _MINIREQUESTOBJECT = _descriptor.Descriptor( @@ -304,7 +304,7 @@ is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='proto', full_name='coprocess.MiniRequestObject.proto', index=12, + name='scheme', full_name='coprocess.MiniRequestObject.scheme', index=12, number=13, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -323,7 +323,7 @@ oneofs=[ ], serialized_start=85, - serialized_end=860, + serialized_end=861, ) _MINIREQUESTOBJECT_HEADERSENTRY.containing_type = _MINIREQUESTOBJECT diff --git a/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb b/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb index 2f479ea0814..afb69231ab6 100644 --- a/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb +++ b/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb @@ -18,7 +18,7 @@ optional :return_overrides, :message, 10, "coprocess.ReturnOverrides" optional :method, :string, 11 optional :request_uri, :string, 12 - optional :proto, :string, 13 + optional :scheme, :string, 13 end end diff --git a/coprocess/coprocess_mini_request_object.pb.go b/coprocess/coprocess_mini_request_object.pb.go index f8deb7c0383..b0dd906e1ab 100644 --- a/coprocess/coprocess_mini_request_object.pb.go +++ b/coprocess/coprocess_mini_request_object.pb.go @@ -25,7 +25,7 @@ type MiniRequestObject struct { ReturnOverrides *ReturnOverrides `protobuf:"bytes,10,opt,name=return_overrides,json=returnOverrides" json:"return_overrides,omitempty"` Method string `protobuf:"bytes,11,opt,name=method" json:"method,omitempty"` RequestUri string `protobuf:"bytes,12,opt,name=request_uri,json=requestUri" json:"request_uri,omitempty"` - Proto string `protobuf:"bytes,13,opt,name=proto" json:"proto,omitempty"` + Scheme string `protobuf:"bytes,13,opt,name=scheme" json:"scheme,omitempty"` } func (m *MiniRequestObject) Reset() { *m = MiniRequestObject{} } @@ -117,9 +117,9 @@ func (m *MiniRequestObject) GetRequestUri() string { return "" } -func (m *MiniRequestObject) GetProto() string { +func (m *MiniRequestObject) GetScheme() string { if m != nil { - return m.Proto + return m.Scheme } return "" } @@ -131,32 +131,32 @@ func init() { func init() { proto.RegisterFile("coprocess_mini_request_object.proto", fileDescriptor1) } var fileDescriptor1 = []byte{ - // 429 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x92, 0xd1, 0x8b, 0xd3, 0x40, - 0x10, 0xc6, 0x49, 0x7b, 0xd7, 0x9a, 0x49, 0xaf, 0x3d, 0x57, 0x91, 0x25, 0x2f, 0x06, 0x0f, 0x21, - 0xa2, 0x04, 0x39, 0x5f, 0xf4, 0x50, 0xf0, 0x90, 0x82, 0x08, 0xc7, 0x49, 0xc4, 0x07, 0x9f, 0x42, - 0xda, 0x1d, 0xb8, 0x78, 0x6d, 0xb6, 0xee, 0x6e, 0x0e, 0xf3, 0x57, 0xfa, 0x2f, 0x49, 0x67, 0x93, - 0x98, 0xd4, 0x23, 0x90, 0xb7, 0x9d, 0x6f, 0xbf, 0xef, 0xc7, 0x30, 0x33, 0x70, 0xb6, 0x96, 0x3b, - 0x25, 0xd7, 0xa8, 0x75, 0xb2, 0xcd, 0xf2, 0x2c, 0x51, 0xf8, 0xab, 0x40, 0x6d, 0x12, 0xb9, 0xfa, - 0x89, 0x6b, 0x13, 0xed, 0x94, 0x34, 0x92, 0xb9, 0x8d, 0xc9, 0x0f, 0xfe, 0xf9, 0x15, 0x9a, 0x42, - 0xe5, 0x89, 0xbc, 0x43, 0xa5, 0x32, 0x81, 0xda, 0x9a, 0x9f, 0xfd, 0x99, 0xc2, 0xc3, 0xab, 0x2c, - 0xcf, 0x62, 0x4b, 0xba, 0x26, 0x10, 0xfb, 0x04, 0xd3, 0x1b, 0x4c, 0x05, 0x2a, 0xcd, 0x9d, 0x60, - 0x1c, 0x7a, 0xe7, 0x2f, 0xa2, 0x86, 0x14, 0xfd, 0x67, 0x8f, 0x3e, 0x5b, 0xef, 0x32, 0x37, 0xaa, - 0x8c, 0xeb, 0x24, 0xbb, 0x02, 0x4f, 0xa3, 0x49, 0x6a, 0xd0, 0x88, 0x40, 0xaf, 0x7a, 0x41, 0xdf, - 0xd0, 0x74, 0x58, 0xa0, 0x1b, 0x81, 0x3d, 0x87, 0xb9, 0xc0, 0x0d, 0x1a, 0x6c, 0x88, 0xe3, 0x60, - 0x1c, 0xba, 0xf1, 0x89, 0x55, 0x6b, 0x1b, 0x83, 0xa3, 0x95, 0x14, 0x25, 0x3f, 0x0a, 0x9c, 0xd0, - 0x8d, 0xe9, 0xcd, 0x4e, 0x61, 0x5c, 0xa8, 0x0d, 0x3f, 0x26, 0x69, 0xff, 0x64, 0x1f, 0x61, 0xb2, - 0x4b, 0x55, 0xba, 0xd5, 0x7c, 0x42, 0x6d, 0x85, 0xbd, 0x6d, 0x7d, 0x25, 0xab, 0x6d, 0xa9, 0xca, - 0xb1, 0x2f, 0x00, 0xa9, 0x10, 0x49, 0x45, 0x99, 0x12, 0xe5, 0x65, 0x2f, 0xe5, 0x52, 0x88, 0x36, - 0xc8, 0x4d, 0xeb, 0x9a, 0xfd, 0x80, 0x05, 0xfe, 0x36, 0x98, 0x0b, 0x6c, 0x80, 0x0f, 0x08, 0xf8, - 0xba, 0x17, 0xb8, 0xac, 0x32, 0x6d, 0xea, 0x1c, 0x3b, 0x22, 0x3b, 0x83, 0x6a, 0x3e, 0x35, 0xd8, - 0xa5, 0xa1, 0xcd, 0xac, 0x58, 0x99, 0x96, 0x70, 0x7a, 0x78, 0x1e, 0x1c, 0x02, 0x27, 0xf4, 0xce, - 0xfd, 0x56, 0x03, 0x31, 0x59, 0xae, 0x6b, 0x47, 0xbc, 0x50, 0x5d, 0x81, 0x3d, 0x81, 0xc9, 0x16, - 0xcd, 0x8d, 0x14, 0xdc, 0xa3, 0x49, 0x57, 0x15, 0x7b, 0x0a, 0x5e, 0x7d, 0xa8, 0x85, 0xca, 0xf8, - 0x8c, 0x3e, 0xa1, 0x92, 0xbe, 0xab, 0x8c, 0x3d, 0x86, 0x63, 0xba, 0x46, 0x7e, 0x42, 0x5f, 0xb6, - 0xf0, 0x2f, 0x60, 0xd6, 0x3e, 0x86, 0xfd, 0x16, 0x6f, 0xb1, 0xe4, 0x8e, 0xdd, 0xe2, 0x2d, 0x96, - 0xfb, 0xdc, 0x5d, 0xba, 0x29, 0x90, 0x8f, 0x6c, 0x8e, 0x8a, 0x8b, 0xd1, 0x5b, 0xc7, 0xff, 0x00, - 0x8b, 0x83, 0x5b, 0x1a, 0x14, 0x7f, 0x07, 0x5e, 0x6b, 0xa8, 0x83, 0xa2, 0xef, 0x61, 0xde, 0x5d, - 0xf4, 0xa0, 0xf4, 0x25, 0x3c, 0xba, 0x67, 0xab, 0x43, 0x10, 0xab, 0x09, 0x4d, 0xef, 0xcd, 0xdf, - 0x00, 0x00, 0x00, 0xff, 0xff, 0x60, 0xc5, 0x9f, 0x29, 0x2c, 0x04, 0x00, 0x00, + // 431 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0xd1, 0x6b, 0xd4, 0x40, + 0x10, 0xc6, 0xc9, 0xa5, 0xcd, 0x99, 0xc9, 0xf5, 0xae, 0xae, 0x22, 0x4b, 0x5e, 0x0c, 0x16, 0x21, + 0xa2, 0x04, 0xa9, 0x2f, 0x5a, 0x14, 0x2c, 0x72, 0x20, 0x42, 0xa9, 0x44, 0x7c, 0xf0, 0x29, 0xe4, + 0xb2, 0x03, 0x17, 0x7b, 0xc9, 0x9e, 0xbb, 0x9b, 0x62, 0xfe, 0x4c, 0xff, 0x23, 0xb9, 0xdd, 0x24, + 0x26, 0xb5, 0x04, 0xf2, 0xb6, 0xf3, 0xe5, 0xfb, 0x7e, 0x0c, 0x33, 0x13, 0x38, 0xcb, 0xf8, 0x5e, + 0xf0, 0x0c, 0xa5, 0x4c, 0x8a, 0xbc, 0xcc, 0x13, 0x81, 0xbf, 0x2a, 0x94, 0x2a, 0xe1, 0x9b, 0x9f, + 0x98, 0xa9, 0x68, 0x2f, 0xb8, 0xe2, 0xc4, 0xed, 0x4c, 0x7e, 0xf0, 0xcf, 0x2f, 0x50, 0x55, 0xa2, + 0x4c, 0xf8, 0x2d, 0x0a, 0x91, 0x33, 0x94, 0xc6, 0xfc, 0xec, 0xcf, 0x1c, 0x1e, 0x5e, 0xe5, 0x65, + 0x1e, 0x1b, 0xd2, 0xb5, 0x06, 0x91, 0x4f, 0x30, 0xdf, 0x62, 0xca, 0x50, 0x48, 0x6a, 0x05, 0x76, + 0xe8, 0x9d, 0xbf, 0x88, 0x3a, 0x52, 0xf4, 0x9f, 0x3d, 0xfa, 0x6c, 0xbc, 0xeb, 0x52, 0x89, 0x3a, + 0x6e, 0x93, 0xe4, 0x0a, 0x3c, 0x89, 0x2a, 0x69, 0x41, 0x33, 0x0d, 0x7a, 0x35, 0x0a, 0xfa, 0x86, + 0x6a, 0xc0, 0x02, 0xd9, 0x09, 0xe4, 0x39, 0x2c, 0x19, 0xee, 0x50, 0x61, 0x47, 0xb4, 0x03, 0x3b, + 0x74, 0xe3, 0x13, 0xa3, 0xb6, 0x36, 0x02, 0x47, 0x1b, 0xce, 0x6a, 0x7a, 0x14, 0x58, 0xa1, 0x1b, + 0xeb, 0x37, 0x39, 0x05, 0xbb, 0x12, 0x3b, 0x7a, 0xac, 0xa5, 0xc3, 0x93, 0x7c, 0x04, 0x67, 0x9f, + 0x8a, 0xb4, 0x90, 0xd4, 0xd1, 0x6d, 0x85, 0xa3, 0x6d, 0x7d, 0xd5, 0x56, 0xd3, 0x52, 0x93, 0x23, + 0x5f, 0x00, 0x52, 0xc6, 0x92, 0x86, 0x32, 0xd7, 0x94, 0x97, 0xa3, 0x94, 0x4b, 0xc6, 0xfa, 0x20, + 0x37, 0x6d, 0x6b, 0xf2, 0x03, 0x56, 0xf8, 0x5b, 0x61, 0xc9, 0xb0, 0x03, 0x3e, 0xd0, 0xc0, 0xd7, + 0xa3, 0xc0, 0x75, 0x93, 0xe9, 0x53, 0x97, 0x38, 0x10, 0xc9, 0x19, 0x34, 0xf3, 0x69, 0xc1, 0xae, + 0x1e, 0xda, 0xc2, 0x88, 0x8d, 0x69, 0x0d, 0xa7, 0x77, 0xcf, 0x83, 0x42, 0x60, 0x85, 0xde, 0xb9, + 0xdf, 0x6b, 0x20, 0xd6, 0x96, 0xeb, 0xd6, 0x11, 0xaf, 0xc4, 0x50, 0x20, 0x4f, 0xc0, 0x29, 0x50, + 0x6d, 0x39, 0xa3, 0x9e, 0x9e, 0x74, 0x53, 0x91, 0xa7, 0xe0, 0xb5, 0x87, 0x5a, 0x89, 0x9c, 0x2e, + 0xf4, 0x47, 0x68, 0xa4, 0xef, 0x22, 0x3f, 0x04, 0x65, 0xb6, 0xc5, 0x02, 0xe9, 0x89, 0x09, 0x9a, + 0xca, 0xbf, 0x80, 0x45, 0xff, 0x1c, 0x0e, 0x7b, 0xbc, 0xc1, 0x9a, 0x5a, 0x66, 0x8f, 0x37, 0x58, + 0x93, 0xc7, 0x70, 0x7c, 0x9b, 0xee, 0x2a, 0xa4, 0x33, 0xad, 0x99, 0xe2, 0x62, 0xf6, 0xd6, 0xf2, + 0x3f, 0xc0, 0xea, 0xce, 0x35, 0x4d, 0x8a, 0xbf, 0x03, 0xaf, 0x37, 0xd6, 0x49, 0xd1, 0xf7, 0xb0, + 0x1c, 0xae, 0x7a, 0x52, 0xfa, 0x12, 0x1e, 0xdd, 0xb3, 0xd7, 0x29, 0x88, 0x8d, 0xa3, 0x7f, 0xed, + 0x37, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x7f, 0x64, 0x62, 0x2e, 0x04, 0x00, 0x00, } diff --git a/coprocess/proto/coprocess_mini_request_object.proto b/coprocess/proto/coprocess_mini_request_object.proto index 8298fbffbc8..c9f18ab63e0 100644 --- a/coprocess/proto/coprocess_mini_request_object.proto +++ b/coprocess/proto/coprocess_mini_request_object.proto @@ -18,5 +18,5 @@ message MiniRequestObject { ReturnOverrides return_overrides = 10; string method = 11; string request_uri = 12; - string proto = 13; + string scheme = 13; } From 9fa81e192b6dc70287d178bc525b814266b1185e Mon Sep 17 00:00:00 2001 From: Ahmet Soormally Date: Thu, 1 Feb 2018 12:18:57 +0000 Subject: [PATCH 22/54] feature: strip path version from proxy upstream resolves: https://github.com/TykTechnologies/tyk/issues/492 https://community.tyk.io/t/api-routing-via-header-option-and-catch-all-routing/2365/2 --- api_definition.go | 5 ++--- apidef/api_definitions.go | 5 +++-- handler_success.go | 22 ++++++++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/api_definition.go b/api_definition.go index 08b38c89847..9afc6bd1ba7 100644 --- a/api_definition.go +++ b/api_definition.go @@ -1033,9 +1033,9 @@ func (a *APISpec) getVersionFromRequest(r *http.Request) string { return r.URL.Query().Get(a.VersionDefinition.Key) case "url": - url := strings.Replace(r.URL.Path, a.Proxy.ListenPath, "", 1) + uPath := strings.Replace(r.URL.Path, a.Proxy.ListenPath, "", 1) // First non-empty part of the path is the version ID - for _, part := range strings.Split(url, "/") { + for _, part := range strings.Split(uPath, "/") { if part != "" { return part } @@ -1158,7 +1158,6 @@ func (a *APISpec) Version(r *http.Request) (*apidef.VersionInfo, []URLSpec, bool } return &version, rxPaths, whiteListStatus, StatusOk - } type RoundRobin struct { diff --git a/apidef/api_definitions.go b/apidef/api_definitions.go index ae73a02e647..d87f7d59cae 100644 --- a/apidef/api_definitions.go +++ b/apidef/api_definitions.go @@ -348,8 +348,9 @@ type APIDefinition struct { HmacAllowedClockSkew float64 `bson:"hmac_allowed_clock_skew" json:"hmac_allowed_clock_skew"` BaseIdentityProvidedBy AuthTypeEnum `bson:"base_identity_provided_by" json:"base_identity_provided_by"` VersionDefinition struct { - Location string `bson:"location" json:"location"` - Key string `bson:"key" json:"key"` + Location string `bson:"location" json:"location"` + Key string `bson:"key" json:"key"` + StripPath bool `bson:"strip_path" json:"strip_path"` } `bson:"definition" json:"definition"` VersionData struct { NotVersioned bool `bson:"not_versioned" json:"not_versioned"` diff --git a/handler_success.go b/handler_success.go index d415432e13e..6287d6fee25 100644 --- a/handler_success.go +++ b/handler_success.go @@ -224,6 +224,17 @@ func recordDetail(r *http.Request, globalConf config.Config) bool { // Spec states the path is Ignored func (s *SuccessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) *http.Response { log.Debug("Started proxy") + + versionDef := s.Spec.VersionDefinition + if versionDef.Location == "url" && versionDef.StripPath { + part := s.Spec.getVersionFromRequest(r) + + log.Info("Stripping version from url: ", part) + + r.URL.Path = strings.Replace(r.URL.Path, part+"/", "", 1) + r.URL.RawPath = strings.Replace(r.URL.RawPath, part+"/", "", 1) + } + // Make sure we get the correct target URL if s.Spec.Proxy.StripListenPath { log.Debug("Stripping: ", s.Spec.Proxy.ListenPath) @@ -262,6 +273,17 @@ func (s *SuccessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) *http // final destination, this is invoked by the ProxyHandler or right at the start of a request chain if the URL // Spec states the path is Ignored Itwill also return a response object for the cache func (s *SuccessHandler) ServeHTTPWithCache(w http.ResponseWriter, r *http.Request) *http.Response { + + versionDef := s.Spec.VersionDefinition + if versionDef.Location == "url" && versionDef.StripPath { + part := s.Spec.getVersionFromRequest(r) + + log.Info("Stripping version from url: ", part) + + r.URL.Path = strings.Replace(r.URL.Path, part+"/", "", 1) + r.URL.RawPath = strings.Replace(r.URL.RawPath, part+"/", "", 1) + } + // Make sure we get the correct target URL if s.Spec.Proxy.StripListenPath { r.URL.Path = strings.Replace(r.URL.Path, s.Spec.Proxy.ListenPath, "", 1) From 28f0b754198833b043024cb937358825ea393be5 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 28 May 2018 18:39:11 +0300 Subject: [PATCH 23/54] Fix coprocess auth when session set and token empty Check if token is empty. Fix https://github.com/TykTechnologies/tyk/issues/1739 --- coprocess.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coprocess.go b/coprocess.go index 3b78758d46a..9fe969156ab 100644 --- a/coprocess.go +++ b/coprocess.go @@ -309,7 +309,7 @@ func (m *CoProcessMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Requ // Is this a CP authentication middleware? if m.Spec.EnableCoProcessAuth && m.HookType == coprocess.HookType_CustomKeyCheck { // The CP middleware didn't setup a session: - if returnObject.Session == nil { + if returnObject.Session == nil || token == "" { authHeaderValue := r.Header.Get(m.Spec.Auth.AuthHeaderName) AuthFailed(m, r, authHeaderValue) return errors.New("Key not authorised"), 403 From 07c958f0ec1519e3a2ed329ddddd3bdaa797f5cb Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 29 May 2018 14:44:25 +0300 Subject: [PATCH 24/54] Try to fix proxy tests --- cert_go1.10_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cert_go1.10_test.go b/cert_go1.10_test.go index 5d2634637a1..2c6c41e23f3 100644 --- a/cert_go1.10_test.go +++ b/cert_go1.10_test.go @@ -189,6 +189,9 @@ func TestProxyTransport(t *testing.T) { }) t.Run("API: Valid proxy", func(t *testing.T) { + globalConf.ProxySSLMinVersion = 771 + config.SetGlobal(globalConf) + _, _, _, proxyCert := genServerCertificate() proxy := initProxy("https", &tls.Config{ Certificates: []tls.Certificate{proxyCert}, From 1c3a4007bd2122d3f62ec68e9184162e2a966b5b Mon Sep 17 00:00:00 2001 From: Ahmet Soormally Date: Fri, 25 May 2018 20:20:37 +0100 Subject: [PATCH 25/54] bug: unable to transform json array response body Create a json response body transform. Use it to filter an array. Because the gateway unmarshals `bodyData` into `map[string]interface{}`, this means that it is unable to unmarshal an array. ``` [May 25 19:49:13] ERROR outbound-transform: Error unmarshalling JSON: json: cannot unmarshal array into Go value of type map[string]interface {} api_id=62498653ade2405b64537d94d2666dc0 path=example-array server_name=https://my-upstream.com/ ``` This fix resolves the response body transform by unmarshaling into an `interface{}`. We need to find a suitable solution for request body transforms. This is currently not possible due to: ``` if tmeta.TemplateData.EnableSession { session := ctxGetSession(r) bodyData["_tyk_meta"] = session.MetaData } if contextVars { bodyData["_tyk_context"] = ctxGetData(r) } ``` --- res_handler_transform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/res_handler_transform.go b/res_handler_transform.go index bfe6aeed43c..c3ab2127b0d 100644 --- a/res_handler_transform.go +++ b/res_handler_transform.go @@ -76,7 +76,7 @@ func (h *ResponseTransformMiddleware) HandleResponse(rw http.ResponseWriter, res defer respBody.Close() // Put into an interface: - var bodyData map[string]interface{} + var bodyData interface{} switch tmeta.TemplateData.Input { case apidef.RequestXML: mxj.XmlCharsetReader = WrappedCharsetReader From 871f428f6c9364c1572c3e6cc356114bc3a3804a Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Wed, 30 May 2018 00:34:01 -0400 Subject: [PATCH 26/54] Enhance Python errors (#1736) * coprocess: enhance Go error handling when using Python plugins * coprocess: extend Python middleware error messages * coprocess: add a helper for simple log messages * coprocess: modify Python dispatcher to use sys.excepthook --- coprocess.go | 3 +++ coprocess/python/dispatcher.py | 33 +++++++++++++++--------------- coprocess/python/gateway.py | 7 ++++++- coprocess/python/tyk/middleware.py | 5 +++-- coprocess_native.go | 9 +++++++- coprocess_python.go | 24 +++++++++++++--------- 6 files changed, 51 insertions(+), 30 deletions(-) diff --git a/coprocess.go b/coprocess.go index 9fe969156ab..549b5b44852 100644 --- a/coprocess.go +++ b/coprocess.go @@ -266,6 +266,9 @@ func (m *CoProcessMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Requ returnObject, err := coProcessor.Dispatch(object) if err != nil { + log.WithFields(logrus.Fields{ + "prefix": "coprocess", + }).WithError(err).Error("Dispatch error") if m.HookType == coprocess.HookType_CustomKeyCheck { return errors.New("Key not authorised"), 403 } else { diff --git a/coprocess/python/dispatcher.py b/coprocess/python/dispatcher.py index fcf0ccf9e73..104c8222bee 100644 --- a/coprocess/python/dispatcher.py +++ b/coprocess/python/dispatcher.py @@ -1,6 +1,5 @@ from glob import glob from os import getcwd, chdir, path -import sys import tyk from tyk.middleware import TykMiddleware @@ -9,11 +8,17 @@ from gateway import TykGateway as tyk +import sys +def except_hook(type, value, traceback): + tyk.log_error("{0}".format(value)) + +sys.excepthook = except_hook + class TykDispatcher: '''A simple dispatcher''' def __init__(self, bundle_root_path): - tyk.log( "Initializing dispatcher", "info" ) + tyk.log("Initializing dispatcher", "info") self.bundle_root_path = bundle_root_path self.bundles = [] self.hook_table = {} @@ -64,18 +69,14 @@ def find_hook(self, api_id, hook_name): raise Exception('Hook is not defined: {0}'.format(hook_name)) def dispatch_hook(self, object_msg): + object = TykCoProcessObject(object_msg) + api_id = object.spec['APIID'] + middleware, hook_handler = self.find_hook(api_id, object.hook_name) try: - object = TykCoProcessObject(object_msg) - api_id = object.spec['APIID'] - middleware, hook_handler = self.find_hook(api_id, object.hook_name) - if hook_handler: - object = middleware.process(hook_handler, object) - else: - tyk.log( "Can't dispatch '{0}', hook is not defined.".format(object.hook_name), "error") - return object.dump() - except: - tyk.log_error( "Can't dispatch, error:" ) - return object_msg + object = middleware.process(hook_handler, object) + except Exception as e: + raise Exception("Hook '{0}' returned an error: {1}".format(object.hook_name, e)) + return object.dump() def dispatch_event(self, event_json): try: @@ -83,8 +84,8 @@ def dispatch_event(self, event_json): api_id = event.spec['APIID'] middleware, hook_handler = self.find_hook(api_id, event.handler_name) middleware.process(hook_handler, event) - except: - tyk.log_error( "Can't dispatch: ") + except Exception as e: + raise Exception("Couldn't dispatch '{0}' event: {1}", event.handler_name, e) def reload(self): - tyk.log( "Reloading event handlers and middlewares.", "info" ) + tyk.log("Reloading event handlers and middlewares.", "info") diff --git a/coprocess/python/gateway.py b/coprocess/python/gateway.py index 9c694722422..8ca5606519b 100644 --- a/coprocess/python/gateway.py +++ b/coprocess/python/gateway.py @@ -27,7 +27,12 @@ def log(cls, msg, level): @classmethod def log_error(cls, *args): excp = exc_info() - if len(args) == 0: + nargs = len(args) + # For simpler errors: + if nargs == 1: + cls.log(args[0], "error") + return + if nargs == 0: cls.log("{0} {1}".format(excp[0], excp[1]), "error") else: cls.log("{0} {1} {2}".format(args[0], excp[0], excp[1]), "error") diff --git a/coprocess/python/tyk/middleware.py b/coprocess/python/tyk/middleware.py index 39035614df9..44b84ec39be 100644 --- a/coprocess/python/tyk/middleware.py +++ b/coprocess/python/tyk/middleware.py @@ -41,8 +41,8 @@ def __init__(self, filepath, bundle_root_path=None): self.module = imp.load_source(filepath, self.mw_path) self.register_handlers() self.cleanup() - except: - tyk.log_error( "Middleware initialization error:" ) + except Exception as e: + tyk.log_error("Middleware initialization error: {0}".format(e)) def register_handlers(self): new_handlers = {} @@ -63,6 +63,7 @@ def build_hooks_and_event_handlers(self): for handler in self.handlers[hook_type]: handler.middleware = self hooks[handler.name] = handler + tyk.log("Loading hook '{0}' ({1})".format(handler.name, self.filepath), "debug") return hooks def cleanup(self): diff --git a/coprocess_native.go b/coprocess_native.go index fae5a655fcd..f7e18a6057c 100644 --- a/coprocess_native.go +++ b/coprocess_native.go @@ -19,6 +19,8 @@ package main import "C" import ( + "errors" + "github.com/golang/protobuf/proto" "github.com/TykTechnologies/tyk/coprocess" @@ -29,7 +31,9 @@ import ( // Dispatch prepares a CoProcessMessage, sends it to the GlobalDispatcher and gets a reply. func (c *CoProcessor) Dispatch(object *coprocess.Object) (*coprocess.Object, error) { - + if GlobalDispatcher == nil { + return nil, errors.New("Dispatcher not initialized") + } var objectMsg []byte switch MessageType { case coprocess.ProtobufMessage: @@ -47,6 +51,9 @@ func (c *CoProcessor) Dispatch(object *coprocess.Object) (*coprocess.Object, err objectPtr.length = C.int(len(objectMsg)) newObjectPtr := (*C.struct_CoProcessMessage)(GlobalDispatcher.Dispatch(unsafe.Pointer(objectPtr))) + if newObjectPtr == nil { + return nil, errors.New("Dispatch error") + } newObjectBytes := C.GoBytes(newObjectPtr.p_data, newObjectPtr.length) diff --git a/coprocess_python.go b/coprocess_python.go index ae797d19779..4e58273f888 100644 --- a/coprocess_python.go +++ b/coprocess_python.go @@ -135,22 +135,21 @@ static struct CoProcessMessage* Python_DispatchHook(struct CoProcessMessage* obj PyObject *args = PyTuple_Pack( 1, PyBytes_FromStringAndSize(object->p_data, object->length) ); PyObject *result = PyObject_CallObject( dispatcher_hook, args ); - if( result == NULL ) { PyErr_Print(); - } else { - PyObject* new_object_msg_item = PyTuple_GetItem( result, 0 ); - char* output = PyBytes_AsString(new_object_msg_item); + PyGILState_Release(gilState); + return NULL; + } + PyObject* new_object_msg_item = PyTuple_GetItem( result, 0 ); + char* output = PyBytes_AsString(new_object_msg_item); - PyObject* new_object_msg_length = PyTuple_GetItem( result, 1 ); - int msg_length = PyLong_AsLong(new_object_msg_length); + PyObject* new_object_msg_length = PyTuple_GetItem( result, 1 ); + int msg_length = PyLong_AsLong(new_object_msg_length); - outputObject->p_data = (void*)output; - outputObject->length = msg_length; - } + outputObject->p_data = (void*)output; + outputObject->length = msg_length; PyGILState_Release(gilState); - return outputObject; } @@ -261,6 +260,11 @@ func PythonNewDispatcher(bundleRootPath string) (coprocess.Dispatcher, error) { // PythonSetEnv sets PYTHONPATH, it's called before initializing the interpreter. func PythonSetEnv(pythonPaths ...string) { + if config.Global().CoProcessOptions.PythonPathPrefix == "" { + log.WithFields(logrus.Fields{ + "prefix": "coprocess", + }).Warning("Python path prefix isn't set (check \"python_path_prefix\" in tyk.conf)") + } CPythonPath := C.CString(strings.Join(pythonPaths, ":")) defer C.free(unsafe.Pointer(CPythonPath)) C.Python_SetEnv(CPythonPath) From 46843a44587775c6b0ceaa712ae476085559a169 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Wed, 30 May 2018 00:50:26 -0400 Subject: [PATCH 27/54] coprocess: return error when proto.Marshal fails (#1727) * coprocess: return error when proto.Marshal fails * coprocess: add additional error checks and user defer for free() calls --- coprocess_native.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/coprocess_native.go b/coprocess_native.go index f7e18a6057c..b0318de08dd 100644 --- a/coprocess_native.go +++ b/coprocess_native.go @@ -34,24 +34,33 @@ func (c *CoProcessor) Dispatch(object *coprocess.Object) (*coprocess.Object, err if GlobalDispatcher == nil { return nil, errors.New("Dispatcher not initialized") } + var objectMsg []byte + var err error switch MessageType { case coprocess.ProtobufMessage: - objectMsg, _ = proto.Marshal(object) + objectMsg, err = proto.Marshal(object) case coprocess.JsonMessage: - objectMsg, _ = json.Marshal(object) + objectMsg, err = json.Marshal(object) + } + if err != nil { + return nil, err } objectMsgStr := string(objectMsg) CObjectStr := C.CString(objectMsgStr) + defer C.free(unsafe.Pointer(CObjectStr)) objectPtr := (*C.struct_CoProcessMessage)(C.malloc(C.size_t(unsafe.Sizeof(C.struct_CoProcessMessage{})))) objectPtr.p_data = unsafe.Pointer(CObjectStr) objectPtr.length = C.int(len(objectMsg)) + defer C.free(unsafe.Pointer(objectPtr)) newObjectPtr := (*C.struct_CoProcessMessage)(GlobalDispatcher.Dispatch(unsafe.Pointer(objectPtr))) - if newObjectPtr == nil { + defer C.free(unsafe.Pointer(newObjectPtr)) + + if newObjectPtr == nil { return nil, errors.New("Dispatch error") } @@ -61,14 +70,13 @@ func (c *CoProcessor) Dispatch(object *coprocess.Object) (*coprocess.Object, err switch MessageType { case coprocess.ProtobufMessage: - proto.Unmarshal(newObjectBytes, newObject) + err = proto.Unmarshal(newObjectBytes, newObject) case coprocess.JsonMessage: - json.Unmarshal(newObjectBytes, newObject) + err = json.Unmarshal(newObjectBytes, newObject) + } + if err != nil { + return nil, err } - - C.free(unsafe.Pointer(CObjectStr)) - C.free(unsafe.Pointer(objectPtr)) - C.free(unsafe.Pointer(newObjectPtr)) return newObject, nil } From 562a4160e71573cf9ab8be9ceab9af3c9db523be Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Wed, 30 May 2018 00:51:53 -0400 Subject: [PATCH 28/54] [WIP] gRPC tests and benchmarks (#1728) * Modify test helpers to allow setting CP configuration block * coprocess: add initial gRPC tests and benchmarks * coprocess: modify existing Python tests to use coprocessConfig --- coprocess_grpc_test.go | 153 ++++++++++++++++++++++++++ coprocess_id_extractor_python_test.go | 4 + coprocess_python_test.go | 6 +- helpers_test.go | 3 + 4 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 coprocess_grpc_test.go diff --git a/coprocess_grpc_test.go b/coprocess_grpc_test.go new file mode 100644 index 00000000000..135d4abb493 --- /dev/null +++ b/coprocess_grpc_test.go @@ -0,0 +1,153 @@ +// +build coprocess +// +build grpc + +package main + +import ( + "encoding/json" + "io/ioutil" + "net" + "net/http" + "testing" + + "context" + + "google.golang.org/grpc" + + "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/coprocess" + "github.com/TykTechnologies/tyk/test" +) + +const ( + grpcListenAddr = ":9999" + grpcListenPath = "tcp://127.0.0.1:9999" + + testHeaderName = "Testheader" + testHeaderValue = "testvalue" +) + +type dispatcher struct{} + +func (d *dispatcher) Dispatch(ctx context.Context, object *coprocess.Object) (*coprocess.Object, error) { + switch object.HookName { + case "testPreHook1": + object.Request.SetHeaders = map[string]string{ + testHeaderName: testHeaderValue, + } + } + return object, nil +} + +func (d *dispatcher) DispatchEvent(ctx context.Context, event *coprocess.Event) (*coprocess.EventReply, error) { + return &coprocess.EventReply{}, nil +} + +func newTestGRPCServer() (s *grpc.Server) { + s = grpc.NewServer() + coprocess.RegisterDispatcherServer(s, &dispatcher{}) + return s +} + +func loadTestGRPCSpec() *APISpec { + spec := buildAndLoadAPI(func(spec *APISpec) { + spec.APIID = "999999" + spec.OrgID = "default" + spec.Auth = apidef.Auth{ + AuthHeaderName: "authorization", + } + spec.VersionData = struct { + NotVersioned bool `bson:"not_versioned" json:"not_versioned"` + DefaultVersion string `bson:"default_version" json:"default_version"` + Versions map[string]apidef.VersionInfo `bson:"versions" json:"versions"` + }{ + NotVersioned: true, + Versions: map[string]apidef.VersionInfo{ + "v1": { + Name: "v1", + }, + }, + } + spec.Proxy.ListenPath = "/grpc-test-api/" + spec.Proxy.StripListenPath = true + spec.CustomMiddleware = apidef.MiddlewareSection{ + Pre: []apidef.MiddlewareDefinition{ + {Name: "testPreHook1"}, + }, + Driver: apidef.GrpcDriver, + } + })[0] + + return spec +} + +func startTykWithGRPC() (*tykTestServer, *grpc.Server) { + // Setup the gRPC server: + listener, _ := net.Listen("tcp", grpcListenAddr) + grpcServer := newTestGRPCServer() + go grpcServer.Serve(listener) + + // Setup Tyk: + cfg := config.CoProcessConfig{ + EnableCoProcess: true, + CoProcessGRPCServer: grpcListenPath, + } + ts := newTykTestServer(tykTestServerConfig{coprocessConfig: cfg}) + + // Load a test API: + loadTestGRPCSpec() + return &ts, grpcServer +} + +func TestGRPCDispatch(t *testing.T) { + ts, grpcServer := startTykWithGRPC() + defer ts.Close() + defer grpcServer.Stop() + + t.Run("Pre Hook with SetHeaders", func(t *testing.T) { + res, err := ts.Run(t, test.TestCase{ + Path: "/grpc-test-api/", + Method: http.MethodGet, + Code: http.StatusOK, + }) + if err != nil { + t.Fatalf("Request failed: %s", err.Error()) + } + data, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("Couldn't read response body: %s", err.Error()) + } + var testResponse testHttpResponse + err = json.Unmarshal(data, &testResponse) + if err != nil { + t.Fatalf("Couldn't unmarshal test response JSON: %s", err.Error()) + } + value, ok := testResponse.Headers[testHeaderName] + if !ok { + t.Fatalf("Header not found, expecting %s", testHeaderName) + } + if value != testHeaderValue { + t.Fatalf("Header value isn't %s", testHeaderValue) + } + }) + +} + +func BenchmarkGRPCDispatch(b *testing.B) { + ts, grpcServer := startTykWithGRPC() + defer ts.Close() + defer grpcServer.Stop() + + b.Run("Pre Hook with SetHeaders", func(b *testing.B) { + path := "/grpc-test-api/" + b.ReportAllocs() + for i := 0; i < b.N; i++ { + ts.Run(b, test.TestCase{ + Path: path, + Method: http.MethodGet, + Code: http.StatusOK, + }) + } + }) +} diff --git a/coprocess_id_extractor_python_test.go b/coprocess_id_extractor_python_test.go index 9715f3201cf..ce2a1b77966 100644 --- a/coprocess_id_extractor_python_test.go +++ b/coprocess_id_extractor_python_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/test" ) @@ -147,6 +148,9 @@ def MyAuthHook(request, session, metadata, spec): // With ID extractor, it should run multiple times (because cache) func TestValueExtractorHeaderSource(t *testing.T) { ts := newTykTestServer(tykTestServerConfig{ + coprocessConfig: config.CoProcessConfig{ + EnableCoProcess: true, + }, delay: 10 * time.Millisecond, }) defer ts.Close() diff --git a/coprocess_python_test.go b/coprocess_python_test.go index 9c9401ad5c8..86bbcd5674c 100644 --- a/coprocess_python_test.go +++ b/coprocess_python_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/test" ) @@ -42,7 +43,10 @@ def MyAuthHook(request, session, metadata, spec): } func TestPythonBundles(t *testing.T) { - ts := newTykTestServer() + ts := newTykTestServer(tykTestServerConfig{ + coprocessConfig: config.CoProcessConfig{ + EnableCoProcess: true, + }}) defer ts.Close() bundleID := registerBundle("python_with_auth_check", pythonBundleWithAuthCheck) diff --git a/helpers_test.go b/helpers_test.go index 732ce6f6b41..11f50511b9f 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -282,6 +282,7 @@ type tykTestServerConfig struct { delay time.Duration hotReload bool overrideDefaults bool + coprocessConfig config.CoProcessConfig } type tykTestServer struct { @@ -306,6 +307,8 @@ func (s *tykTestServer) Start() { globalConf.ControlAPIPort, _ = strconv.Atoi(port) } + globalConf.CoProcessOptions = s.config.coprocessConfig + config.SetGlobal(globalConf) setupGlobals() From d7a606cc4d171f9fcd32dc9e0c6709cd96ed4ac0 Mon Sep 17 00:00:00 2001 From: Ahmet Soormally Date: Fri, 18 May 2018 11:30:21 +0100 Subject: [PATCH 29/54] bug: EOF when performing response body transform fixes: #1729 Description of bug: 1. Create an API with a mock XML response e.g. `/api-a/mock`. 2. Create an API which reverse proxies to `api-a` called `abi-b`. 3. perform a response body transform on `api-b/mock` endpoint. e.g. convert to json. Bug: ``` ERROR outbound-transform: Error unmarshalling XML: xml.Decoder.Token() - XML syntax error on line 167: unexpected EOF api_id=90a1e135513946cb437b5e5111422227 path=soap server_name=https://tyk-gateway.dev:8080/api-a/ ``` Fix: In the case that we are reading XML, we use `ioutil.ReadAll` to read the response body, and pass that to `mxj.NewMapXml`. Lint: cleaning up repetition of `logrus.Fields` --- res_handler_transform.go | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/res_handler_transform.go b/res_handler_transform.go index c3ab2127b0d..7e3ceb9d329 100644 --- a/res_handler_transform.go +++ b/res_handler_transform.go @@ -27,6 +27,7 @@ func (h *ResponseTransformMiddleware) Init(c interface{}, spec *APISpec) error { } func respBodyReader(req *http.Request, resp *http.Response) io.ReadCloser { + if req.Header.Get("Accept-Encoding") == "" { return resp.Body } @@ -64,6 +65,14 @@ func compressBuffer(in bytes.Buffer, encoding string) (out bytes.Buffer) { } func (h *ResponseTransformMiddleware) HandleResponse(rw http.ResponseWriter, res *http.Response, req *http.Request, ses *user.SessionState) error { + + logger := log.WithFields(logrus.Fields{ + "prefix": "outbound-transform", + "server_name": h.Spec.Proxy.TargetURL, + "api_id": h.Spec.APIID, + "path": req.URL.Path, + }) + _, versionPaths, _, _ := h.Spec.Version(req) found, meta := h.Spec.CheckSpecMatchesStatus(req, versionPaths, TransformedResponse) @@ -82,35 +91,27 @@ func (h *ResponseTransformMiddleware) HandleResponse(rw http.ResponseWriter, res mxj.XmlCharsetReader = WrappedCharsetReader var err error - bodyData, err = mxj.NewMapXmlReader(respBody) // unmarshal + bodyBytes, err := ioutil.ReadAll(respBody) + if err != nil { + logger.WithError(err).Error("Error reading XML body") + // not returning error, just break out of switch. + break + } + + bodyData, err = mxj.NewMapXml(bodyBytes) // unmarshal if err != nil { - log.WithFields(logrus.Fields{ - "prefix": "outbound-transform", - "server_name": h.Spec.Proxy.TargetURL, - "api_id": h.Spec.APIID, - "path": req.URL.Path, - }).Error("Error unmarshalling XML: ", err) + logger.WithError(err).Error("Error unmarshalling XML") } default: // apidef.RequestJSON if err := json.NewDecoder(respBody).Decode(&bodyData); err != nil { - log.WithFields(logrus.Fields{ - "prefix": "outbound-transform", - "server_name": h.Spec.Proxy.TargetURL, - "api_id": h.Spec.APIID, - "path": req.URL.Path, - }).Error("Error unmarshalling JSON: ", err) + logger.WithError(err).Error("Error unmarshalling JSON") } } // Apply to template var bodyBuffer bytes.Buffer if err := tmeta.Template.Execute(&bodyBuffer, bodyData); err != nil { - log.WithFields(logrus.Fields{ - "prefix": "outbound-transform", - "server_name": h.Spec.Proxy.TargetURL, - "api_id": h.Spec.APIID, - "path": req.URL.Path, - }).Error("Failed to apply template to request: ", err) + logger.WithError(err).Error("Failed to apply template to request") } // Re-compress if original upstream response was compressed From d40fe486f2d6f3dd1f8e5439c01e4cfeff949fba Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Tue, 29 May 2018 12:00:57 -0400 Subject: [PATCH 30/54] reverse_proxy: extend tests and add benchmarks --- reverse_proxy_test.go | 84 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/reverse_proxy_test.go b/reverse_proxy_test.go index ae9ee8b0426..ed64af7d556 100644 --- a/reverse_proxy_test.go +++ b/reverse_proxy_test.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/http/httptest" "net/url" "strings" "testing" @@ -45,10 +46,10 @@ func TestReverseProxyRetainHost(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - spec := &APISpec{APIDefinition: &apidef.APIDefinition{}} + spec := &APISpec{APIDefinition: &apidef.APIDefinition{}, URLRewriteEnabled: true} spec.URLRewriteEnabled = true - req := testReq(t, "GET", tc.inURL, nil) + req := testReq(t, http.MethodGet, tc.inURL, nil) req.URL.Path = tc.inPath if tc.retainHost { setCtxValue(req, RetainHost, true) @@ -63,6 +64,50 @@ func TestReverseProxyRetainHost(t *testing.T) { } } +func testNewWrappedServeHTTP() *ReverseProxy { + target, _ := url.Parse(testHttpGet) + def := apidef.APIDefinition{} + def.VersionData.DefaultVersion = "Default" + def.VersionData.Versions = map[string]apidef.VersionInfo{ + "Default": { + Name: "v2", + UseExtendedPaths: true, + ExtendedPaths: apidef.ExtendedPathsSet{ + TransformHeader: []apidef.HeaderInjectionMeta{ + { + DeleteHeaders: []string{"header"}, + AddHeaders: map[string]string{"newheader": "newvalue"}, + Path: "/abc", + Method: "GET", + ActOnResponse: true, + }, + }, + URLRewrite: []apidef.URLRewriteMeta{ + { + Path: "/get", + Method: "GET", + MatchPattern: "/get", + RewriteTo: "/post", + }, + }, + }, + }, + } + spec := &APISpec{ + APIDefinition: &def, + EnforcedTimeoutEnabled: true, + CircuitBreakerEnabled: true, + } + return TykNewSingleHostReverseProxy(target, spec) +} + +func TestWrappedServeHTTP(t *testing.T) { + proxy := testNewWrappedServeHTTP() + recorder := httptest.NewRecorder() + req, _ := http.NewRequest(http.MethodGet, "/", nil) + proxy.WrappedServeHTTP(recorder, req, false) +} + func TestSingleJoiningSlash(t *testing.T) { tests := []struct { a, b, want string @@ -215,7 +260,7 @@ func TestCheckHeaderInRemoveList(t *testing.T) { for _, tc := range tests { t.Run(fmt.Sprintf("%s:%t", tc.header, tc.expected), func(t *testing.T) { rp := &ReverseProxy{} - r, err := http.NewRequest("GET", "http://test/test", nil) + r, err := http.NewRequest(http.MethodGet, "http://test/test", nil) if err != nil { t.Fatal(err) } @@ -234,6 +279,39 @@ func TestCheckHeaderInRemoveList(t *testing.T) { } } +func testRequestIPHops(t testing.TB) { + req := &http.Request{ + Header: http.Header{}, + RemoteAddr: "test.com:80", + } + req.Header.Set("X-Forwarded-For", "abc") + match := "abc, test.com" + clientIP := requestIPHops(req) + if clientIP != match { + t.Fatalf("Got %s, expected %s", clientIP, match) + } +} + +func TestRequestIPHops(t *testing.T) { + testRequestIPHops(t) +} + +func BenchmarkRequestIPHops(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + testRequestIPHops(b) + } +} + +func BenchmarkWrappedServeHTTP(b *testing.B) { + b.ReportAllocs() + proxy := testNewWrappedServeHTTP() + recorder := httptest.NewRecorder() + req, _ := http.NewRequest(http.MethodGet, "/", nil) + for i := 0; i < b.N; i++ { + proxy.WrappedServeHTTP(recorder, req, false) + } +} func BenchmarkCopyRequestResponse(b *testing.B) { b.ReportAllocs() From 456d9bc58e076eb856c21eb68b25de838a16085e Mon Sep 17 00:00:00 2001 From: Ahmet Soormally Date: Fri, 30 Mar 2018 09:30:59 +0100 Subject: [PATCH 31/54] lint: use mainLog for prefix, creating coprocessLog --- api_loader.go | 162 ++++++++++++++++---------------------------------- 1 file changed, 52 insertions(+), 110 deletions(-) diff --git a/api_loader.go b/api_loader.go index af7c3693ca7..a4c927db0e7 100644 --- a/api_loader.go +++ b/api_loader.go @@ -43,8 +43,7 @@ func prepareStorage() (storage.RedisCluster, storage.RedisCluster, storage.Redis func skipSpecBecauseInvalid(spec *APISpec) bool { if spec.Proxy.ListenPath == "" { - log.WithFields(logrus.Fields{ - "prefix": "main", + mainLog.WithFields(logrus.Fields{ "org_id": spec.OrgID, "api_id": spec.APIID, }).Error("Listen path is empty, skipping API ID: ", spec.APIID) @@ -52,8 +51,7 @@ func skipSpecBecauseInvalid(spec *APISpec) bool { } if strings.Contains(spec.Proxy.ListenPath, " ") { - log.WithFields(logrus.Fields{ - "prefix": "main", + mainLog.WithFields(logrus.Fields{ "org_id": spec.OrgID, "api_id": spec.APIID, }).Error("Listen path contains spaces, is invalid, skipping API ID: ", spec.APIID) @@ -62,11 +60,10 @@ func skipSpecBecauseInvalid(spec *APISpec) bool { _, err := url.Parse(spec.Proxy.TargetURL) if err != nil { - log.WithFields(logrus.Fields{ - "prefix": "main", + mainLog.WithFields(logrus.Fields{ "org_id": spec.OrgID, "api_id": spec.APIID, - }).Error("Couldn't parse target URL: ", err) + }).Error("couldn't parse target URL: ", err) return true } @@ -87,8 +84,7 @@ func countApisByListenHash(specs []*APISpec) map[string]int { if dN == "" { dN = "(no host)" } - log.WithFields(logrus.Fields{ - "prefix": "main", + mainLog.WithFields(logrus.Fields{ "api_name": spec.Name, "domain": dN, }).Info("Tracking hostname") @@ -105,8 +101,12 @@ func processSpec(spec *APISpec, apisByListen map[string]int, var chainDef ChainObject chainDef.Subrouter = subrouter - log.WithFields(logrus.Fields{ - "prefix": "main", + var coprocessLog = log.WithFields(logrus.Fields{ + "prefix": "coprocess", + "api_name": spec.Name, + }) + + mainLog.WithFields(logrus.Fields{ "api_name": spec.Name, }).Info("Loading API") @@ -121,8 +121,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int, } if skipSpecBecauseInvalid(spec) { - log.WithFields(logrus.Fields{ - "prefix": "main", + mainLog.WithFields(logrus.Fields{ "api_name": spec.Name, }).Warning("Skipped!") chainDef.Skip = true @@ -151,8 +150,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int, } } if pathModified { - log.WithFields(logrus.Fields{ - "prefix": "main", + mainLog.WithFields(logrus.Fields{ "org_id": spec.OrgID, "api_id": spec.APIID, }).Error("Listen path collision, changed to ", spec.Proxy.ListenPath) @@ -202,10 +200,9 @@ func processSpec(spec *APISpec, apisByListen map[string]int, loadBundle(spec) } - // TODO: use config.Global().EnableCoProcess + // TODO: use config.Global.EnableCoProcess if config.Global().EnableJSVM || EnableCoProcess { - log.WithFields(logrus.Fields{ - "prefix": "main", + mainLog.WithFields(logrus.Fields{ "api_name": spec.Name, }).Debug("Loading Middleware") @@ -247,8 +244,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int, var proxy ReturningHttpHandler if enableVersionOverrides { - log.WithFields(logrus.Fields{ - "prefix": "main", + mainLog.WithFields(logrus.Fields{ "api_name": spec.Name, }).Info("Multi target enabled") proxy = NewMultiTargetProxy(spec) @@ -277,8 +273,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int, if spec.UseKeylessAccess { chainDef.Open = true - log.WithFields(logrus.Fields{ - "prefix": "main", + mainLog.WithFields(logrus.Fields{ "api_name": spec.Name, }).Info("Checking security policy: Open") @@ -288,10 +283,8 @@ func processSpec(spec *APISpec, apisByListen map[string]int, for _, obj := range mwPreFuncs { if mwDriver != apidef.OttoDriver { - log.WithFields(logrus.Fields{ - "prefix": "coprocess", - "api_name": spec.Name, - }).Debug("Registering coprocess middleware, hook name: ", obj.Name, "hook type: Pre", ", driver: ", mwDriver) + + coprocessLog.Debug("Registering coprocess middleware, hook name: ", obj.Name, "hook type: Pre", ", driver: ", mwDriver) mwAppendEnabled(&chainArray, &CoProcessMiddleware{baseMid, coprocess.HookType_Pre, obj.Name, mwDriver}) } else { chainArray = append(chainArray, createDynamicMiddleware(obj.Name, true, obj.RequireSession, baseMid)) @@ -320,10 +313,8 @@ func processSpec(spec *APISpec, apisByListen map[string]int, for _, obj := range mwPostFuncs { if mwDriver != apidef.OttoDriver { - log.WithFields(logrus.Fields{ - "prefix": "coprocess", - "api_name": spec.Name, - }).Debug("Registering coprocess middleware, hook name: ", obj.Name, "hook type: Post", ", driver: ", mwDriver) + + coprocessLog.Debug("Registering coprocess middleware, hook name: ", obj.Name, "hook type: Post", ", driver: ", mwDriver) mwAppendEnabled(&chainArray, &CoProcessMiddleware{baseMid, coprocess.HookType_Post, obj.Name, mwDriver}) } else { chainArray = append(chainArray, createDynamicMiddleware(obj.Name, false, obj.RequireSession, baseMid)) @@ -342,10 +333,8 @@ func processSpec(spec *APISpec, apisByListen map[string]int, // Add pre-process MW for _, obj := range mwPreFuncs { if mwDriver != apidef.OttoDriver { - log.WithFields(logrus.Fields{ - "prefix": "coprocess", - "api_name": spec.Name, - }).Debug("Registering coprocess middleware, hook name: ", obj.Name, "hook type: Pre", ", driver: ", mwDriver) + + coprocessLog.Debug("Registering coprocess middleware, hook name: ", obj.Name, "hook type: Pre", ", driver: ", mwDriver) mwAppendEnabled(&chainArray, &CoProcessMiddleware{baseMid, coprocess.HookType_Pre, obj.Name, mwDriver}) } else { chainArray = append(chainArray, createDynamicMiddleware(obj.Name, true, obj.RequireSession, baseMid)) @@ -364,38 +353,23 @@ func processSpec(spec *APISpec, apisByListen map[string]int, // Select the keying method to use for setting session states if mwAppendEnabled(&authArray, &Oauth2KeyExists{baseMid}) { - log.WithFields(logrus.Fields{ - "prefix": "main", - "api_name": spec.Name, - }).Info("Checking security policy: OAuth") + mainLog.WithField("api_name", spec.Name).Info("Checking security policy: OAuth") } if mwAppendEnabled(&authArray, &BasicAuthKeyIsValid{baseMid}) { - log.WithFields(logrus.Fields{ - "prefix": "main", - "api_name": spec.Name, - }).Info("Checking security policy: Basic") + mainLog.WithField("api_name", spec.Name).Info("Checking security policy: Basic") } if mwAppendEnabled(&authArray, &HMACMiddleware{BaseMiddleware: baseMid}) { - log.WithFields(logrus.Fields{ - "prefix": "main", - "api_name": spec.Name, - }).Info("Checking security policy: HMAC") + mainLog.WithField("api_name", spec.Name).Info("Checking security policy: HMAC") } if mwAppendEnabled(&authArray, &JWTMiddleware{baseMid}) { - log.WithFields(logrus.Fields{ - "prefix": "main", - "api_name": spec.Name, - }).Info("Checking security policy: JWT") + mainLog.WithField("api_name", spec.Name).Info("Checking security policy: JWT") } if mwAppendEnabled(&authArray, &OpenIDMW{BaseMiddleware: baseMid}) { - log.WithFields(logrus.Fields{ - "prefix": "main", - "api_name": spec.Name, - }).Info("Checking security policy: OpenID") + mainLog.WithField("api_name", spec.Name).Info("Checking security policy: OpenID") } coprocessAuth := EnableCoProcess && mwDriver != apidef.OttoDriver && spec.EnableCoProcessAuth @@ -403,38 +377,29 @@ func processSpec(spec *APISpec, apisByListen map[string]int, if coprocessAuth { // TODO: check if mwAuthCheckFunc is available/valid - log.WithFields(logrus.Fields{ - "prefix": "coprocess", - "api_name": spec.Name, - }).Debug("Registering coprocess middleware, hook name: ", mwAuthCheckFunc.Name, "hook type: CustomKeyCheck", ", driver: ", mwDriver) + coprocessLog.Debug("Registering coprocess middleware, hook name: ", mwAuthCheckFunc.Name, "hook type: CustomKeyCheck", ", driver: ", mwDriver) newExtractor(spec, baseMid) mwAppendEnabled(&authArray, &CoProcessMiddleware{baseMid, coprocess.HookType_CustomKeyCheck, mwAuthCheckFunc.Name, mwDriver}) } if ottoAuth { - log.WithFields(logrus.Fields{ - "prefix": "main", - }).Info("----> Checking security policy: JS Plugin") + + mainLog.Info("----> Checking security policy: JS Plugin") authArray = append(authArray, createDynamicMiddleware(mwAuthCheckFunc.Name, true, false, baseMid)) } if spec.UseStandardAuth || len(authArray) == 0 { - log.WithFields(logrus.Fields{ - "prefix": "main", - "api_name": spec.Name, - }).Info("Checking security policy: Token") + mainLog.WithField("api_name", spec.Name).Info("Checking security policy: Token") authArray = append(authArray, createMiddleware(&AuthKey{baseMid})) } chainArray = append(chainArray, authArray...) for _, obj := range mwPostAuthCheckFuncs { - log.WithFields(logrus.Fields{ - "prefix": "coprocess", - "api_name": spec.Name, - }).Debug("Registering coprocess middleware, hook name: ", obj.Name, "hook type: Pre", ", driver: ", mwDriver) + + coprocessLog.Debug("Registering coprocess middleware, hook name: ", obj.Name, "hook type: Pre", ", driver: ", mwDriver) mwAppendEnabled(&chainArray, &CoProcessMiddleware{baseMid, coprocess.HookType_PostKeyAuth, obj.Name, mwDriver}) } @@ -455,20 +420,15 @@ func processSpec(spec *APISpec, apisByListen map[string]int, for _, obj := range mwPostFuncs { if mwDriver != apidef.OttoDriver { - log.WithFields(logrus.Fields{ - "prefix": "coprocess", - "api_name": spec.Name, - }).Debug("Registering coprocess middleware, hook name: ", obj.Name, "hook type: Post", ", driver: ", mwDriver) + + coprocessLog.Debug("Registering coprocess middleware, hook name: ", obj.Name, "hook type: Post", ", driver: ", mwDriver) mwAppendEnabled(&chainArray, &CoProcessMiddleware{baseMid, coprocess.HookType_Post, obj.Name, mwDriver}) } else { chainArray = append(chainArray, createDynamicMiddleware(obj.Name, false, obj.RequireSession, baseMid)) } } - log.WithFields(logrus.Fields{ - "prefix": "main", - "api_name": spec.Name, - }).Debug("Custom middleware completed processing") + mainLog.WithField("api_name", spec.Name).Debug("Custom middleware completed processing") // Use createMiddleware(&ModifiedMiddleware{baseMid}) to run custom middleware chain = alice.New(chainArray...).Then(&DummyProxyHandler{SH: SuccessHandler{baseMid}}) @@ -485,19 +445,15 @@ func processSpec(spec *APISpec, apisByListen map[string]int, mwAppendEnabled(&simpleArray, &AccessRightsCheck{baseMid}) rateLimitPath := spec.Proxy.ListenPath + "tyk/rate-limits/" - log.WithFields(logrus.Fields{ - "prefix": "main", - "api_name": spec.Name, - }).Debug("Rate limit endpoint is: ", rateLimitPath) + + mainLog.WithField("api_name", spec.Name).Debug("Rate limit endpoint is: ", rateLimitPath) + chainDef.RateLimitPath = rateLimitPath chainDef.RateLimitChain = alice.New(simpleArray...). Then(http.HandlerFunc(userRatesCheck)) } - log.WithFields(logrus.Fields{ - "prefix": "main", - "api_name": spec.Name, - }).Debug("Setting Listen Path: ", spec.Proxy.ListenPath) + mainLog.WithField("api_name", spec.Name).Debug("Setting Listen Path: ", spec.Proxy.ListenPath) chainDef.ThisHandler = chain chainDef.ListenOn = spec.Proxy.ListenPath + "{rest:.*}" @@ -534,9 +490,7 @@ func loadGlobalApps(router *mux.Router) { loadApps(specs, router) if config.Global().NewRelic.AppName != "" { - log.WithFields(logrus.Fields{ - "prefix": "main", - }).Info("Adding NewRelic instrumentation") + mainLog.Info("Adding NewRelic instrumentation") AddNewRelicInstrumentation(NewRelicApplication, router) } } @@ -546,14 +500,10 @@ func loadApps(specs []*APISpec, muxer *mux.Router) { hostname := config.Global().HostName if hostname != "" { muxer = muxer.Host(hostname).Subrouter() - log.WithFields(logrus.Fields{ - "prefix": "main", - }).Info("API hostname set: ", hostname) + mainLog.Info("API hostname set: ", hostname) } - // load the APi defs - log.WithFields(logrus.Fields{ - "prefix": "main", - }).Info("Loading API configurations.") + + mainLog.Info("Loading API configurations.") tmpSpecRegister := make(map[string]*APISpec) @@ -599,10 +549,7 @@ func loadApps(specs []*APISpec, muxer *mux.Router) { if hostRouters[host] != nil { continue // already set up a subrouter } - log.WithFields(logrus.Fields{ - "prefix": "main", - "domain": host, - }).Info("Sub-router created for domain") + mainLog.WithField("domain", host).Info("Sub-router created for domain") hostRouters[host] = muxer.Host(host).Subrouter() } @@ -610,8 +557,7 @@ func loadApps(specs []*APISpec, muxer *mux.Router) { go func(spec *APISpec, i int) { subrouter := hostRouters[spec.Domain] if subrouter == nil { - log.WithFields(logrus.Fields{ - "prefix": "main", + mainLog.WithFields(logrus.Fields{ "domain": spec.Domain, "api_id": spec.APIID, }).Warning("Trying to load API with Domain when custom domains are disabled.") @@ -640,16 +586,14 @@ func loadApps(specs []*APISpec, muxer *mux.Router) { chainObj.Subrouter.Handle(chainObj.RateLimitPath, chainObj.RateLimitChain) } - log.WithFields(logrus.Fields{ - "prefix": "main", - }).Info("Processed and listening on: ", chainObj.ListenOn) + mainLog.Info("Processed and listening on: ", chainObj.ListenOn) chainObj.Subrouter.Handle(chainObj.ListenOn, chainObj.ThisHandler) } // All APIs processed, now we can healthcheck // Add a root message to check all is OK muxer.HandleFunc("/hello", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "Hello Tiki") + fmt.Fprint(w, "Hello Tiki") }) // Swap in the new register @@ -657,16 +601,14 @@ func loadApps(specs []*APISpec, muxer *mux.Router) { apisByID = tmpSpecRegister apisMu.Unlock() - log.Debug("Checker host list") + mainLog.Debug("Checker host list") // Kick off our host checkers if !config.Global().UptimeTests.Disable { SetCheckerHostList() } - log.Debug("Checker host Done") + mainLog.Debug("Checker host Done") - log.WithFields(logrus.Fields{ - "prefix": "main", - }).Info("Initialised API Definitions") + mainLog.Info("Initialised API Definitions") } From 99f81efb810d0823b3225894d19048ca6332978b Mon Sep 17 00:00:00 2001 From: Ahmet Soormally Date: Fri, 30 Mar 2018 09:43:31 +0100 Subject: [PATCH 32/54] lint: pubSubLog to avoid repetition of log.WithFields... --- main.go | 1 + redis_signals.go | 83 +++++++++++++++++++++++------------------------- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/main.go b/main.go index 72eba79052e..beb15a27e83 100644 --- a/main.go +++ b/main.go @@ -51,6 +51,7 @@ import ( var ( log = logger.Get() mainLog = log.WithField("prefix", "main") + pubSubLog = log.WithField("prefix", "pub-sub") rawLog = logger.GetRaw() templates *template.Template analytics RedisAnalyticsHandler diff --git a/redis_signals.go b/redis_signals.go index 436f52f6f1a..7d5cb069d80 100644 --- a/redis_signals.go +++ b/redis_signals.go @@ -8,7 +8,6 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/garyburd/redigo/redis" "github.com/TykTechnologies/goverify" @@ -51,17 +50,11 @@ func startPubSubLoop() { handleRedisEvent(v, nil, nil) }) if err != nil { - log.WithFields(logrus.Fields{ - "prefix": "pub-sub", - "err": err, - }).Error("Connection to Redis failed, reconnect in 10s") + pubSubLog.WithField("err", err).Error("Connection to Redis failed, reconnect in 10s") time.Sleep(10 * time.Second) - log.WithFields(logrus.Fields{ - "prefix": "pub-sub", - }).Warning("Reconnecting") + pubSubLog.Warning("Reconnecting") } - } } @@ -72,7 +65,7 @@ func handleRedisEvent(v interface{}, handled func(NotificationCommand), reloaded } notif := Notification{} if err := json.Unmarshal(message.Data, ¬if); err != nil { - log.Error("Unmarshalling message body failed, malformed: ", err) + pubSubLog.Error("Unmarshalling message body failed, malformed: ", err) return } @@ -84,9 +77,7 @@ func handleRedisEvent(v interface{}, handled func(NotificationCommand), reloaded // Check for a signature, if not signature found, handle if !isPayloadSignatureValid(notif) { - log.WithFields(logrus.Fields{ - "prefix": "pub-sub", - }).Error("Payload signature is invalid!") + pubSubLog.Error("Payload signature is invalid!") return } @@ -108,16 +99,12 @@ func handleRedisEvent(v interface{}, handled func(NotificationCommand), reloaded case NoticeGatewayLENotification: onLESSLStatusReceivedHandler(notif.Payload) case NoticeApiUpdated, NoticeApiRemoved, NoticeApiAdded, NoticePolicyChanged, NoticeGroupReload: - log.WithFields(logrus.Fields{ - "prefix": "pub-sub", - }).Info("Reloading endpoints") + pubSubLog.Info("Reloading endpoints") reloadURLStructure(reloaded) case KeySpaceUpdateNotification: handleKeySpaceEventCacheFlush(notif.Payload) default: - log.WithFields(logrus.Fields{ - "prefix": "pub-sub", - }).Warnf("Unknown notification command: %q", notif.Command) + pubSubLog.Warnf("Unknown notification command: %q", notif.Command) return } if handled != nil { @@ -127,6 +114,7 @@ func handleRedisEvent(v interface{}, handled func(NotificationCommand), reloaded } func handleKeySpaceEventCacheFlush(payload string) { + keys := strings.Split(payload, ",") for _, key := range keys { @@ -144,6 +132,7 @@ var redisInsecureWarn sync.Once var notificationVerifier goverify.Verifier func isPayloadSignatureValid(notification Notification) bool { + switch notification.Command { case NoticeGatewayDRLNotification, NoticeGatewayLENotification: // Gateway to gateway @@ -152,10 +141,9 @@ func isPayloadSignatureValid(notification Notification) bool { if notification.Signature == "" && config.Global().AllowInsecureConfigs { redisInsecureWarn.Do(func() { - log.WithFields(logrus.Fields{ - "prefix": "pub-sub", - }).Warning("Insecure configuration detected (allowing)!") + pubSubLog.Warning("Insecure configuration detected (allowing)!") }) + return true } @@ -163,25 +151,24 @@ func isPayloadSignatureValid(notification Notification) bool { var err error notificationVerifier, err = goverify.LoadPublicKeyFromFile(config.Global().PublicKeyPath) if err != nil { - log.WithFields(logrus.Fields{ - "prefix": "pub-sub", - }).Error("Notification signer: Failed loading private key from path: ", err) + + pubSubLog.Error("Notification signer: Failed loading private key from path: ", err) return false } } if notificationVerifier != nil { + signed, err := base64.StdEncoding.DecodeString(notification.Signature) if err != nil { - log.WithFields(logrus.Fields{ - "prefix": "pub-sub", - }).Error("Failed to decode signature: ", err) + + pubSubLog.Error("Failed to decode signature: ", err) return false } + if err := notificationVerifier.Verify([]byte(notification.Payload), signed); err != nil { - log.WithFields(logrus.Fields{ - "prefix": "pub-sub", - }).Error("Could not verify notification: ", err, ": ", notification) + + pubSubLog.Error("Could not verify notification: ", err, ": ", notification) return false } @@ -200,16 +187,23 @@ type RedisNotifier struct { // Notify will send a notification to a channel func (r *RedisNotifier) Notify(notif interface{}) bool { + toSend, err := json.Marshal(notif) + if err != nil { - log.Error("Problem marshalling notification: ", err) + + pubSubLog.Error("Problem marshalling notification: ", err) return false } - log.Debug("Sending notification", notif) + + pubSubLog.Debug("Sending notification", notif) + if err := r.store.Publish(r.channel, string(toSend)); err != nil { - log.Error("Could not send notification: ", err) + + pubSubLog.Error("Could not send notification: ", err) return false } + return true } @@ -223,26 +217,31 @@ type dashboardConfigPayload struct { } func createConnectionStringFromDashboardObject(config dashboardConfigPayload) string { + hostname := "http://" + if config.DashboardConfig.UseTLS { hostname = "https://" } + hostname += config.DashboardConfig.Hostname if config.DashboardConfig.Port != 0 { + hostname = strings.TrimRight(hostname, "/") hostname += ":" + strconv.Itoa(config.DashboardConfig.Port) } + return hostname } func handleDashboardZeroConfMessage(payload string) { // Decode the configuration from the payload dashPayload := dashboardConfigPayload{} + if err := json.Unmarshal([]byte(payload), &dashPayload); err != nil { - log.WithFields(logrus.Fields{ - "prefix": "pub-sub", - }).Error("Failed to decode dashboard zeroconf payload") + + pubSubLog.Error("Failed to decode dashboard zeroconf payload") return } @@ -252,7 +251,8 @@ func handleDashboardZeroConfMessage(payload string) { hostname := createConnectionStringFromDashboardObject(dashPayload) setHostname := false - globalConf := config.Global() + + globalConf := config.Global() if globalConf.DBAppConfOptions.ConnectionString == "" { globalConf.DBAppConfOptions.ConnectionString = hostname setHostname = true @@ -265,11 +265,6 @@ func handleDashboardZeroConfMessage(payload string) { if setHostname { config.SetGlobal(globalConf) - } - - if setHostname { - log.WithFields(logrus.Fields{ - "prefix": "pub-sub", - }).Info("Hostname set with dashboard zeroconf signal") + pubSubLog.Info("Hostname set with dashboard zeroconf signal") } } From 76f4e7f5ab5b6e6ddef02b0d4a0d74832d023036 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 30 May 2018 08:08:28 +0300 Subject: [PATCH 33/54] Fix typos and formatting --- redis_signals.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/redis_signals.go b/redis_signals.go index 7d5cb069d80..1e0e3dd5cee 100644 --- a/redis_signals.go +++ b/redis_signals.go @@ -149,6 +149,10 @@ func isPayloadSignatureValid(notification Notification) bool { if config.Global().PublicKeyPath != "" && notificationVerifier == nil { var err error +<<<<<<< HEAD +======= + +>>>>>>> Fix typos and formatting notificationVerifier, err = goverify.LoadPublicKeyFromFile(config.Global().PublicKeyPath) if err != nil { @@ -245,14 +249,15 @@ func handleDashboardZeroConfMessage(payload string) { return } - if !config.Global().UseDBAppConfigs || config.Global().DisableDashboardZeroConf { + globalConf := config.Global() + + if !globalConf.UseDBAppConfigs || globalConf.DisableDashboardZeroConf { return } hostname := createConnectionStringFromDashboardObject(dashPayload) setHostname := false - globalConf := config.Global() if globalConf.DBAppConfOptions.ConnectionString == "" { globalConf.DBAppConfOptions.ConnectionString = hostname setHostname = true From 89149aeeced343f3ab5280d6f2c613398071e14e Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 30 May 2018 08:19:57 +0300 Subject: [PATCH 34/54] Fix typos --- redis_signals.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/redis_signals.go b/redis_signals.go index 1e0e3dd5cee..2875e14c9e7 100644 --- a/redis_signals.go +++ b/redis_signals.go @@ -149,10 +149,7 @@ func isPayloadSignatureValid(notification Notification) bool { if config.Global().PublicKeyPath != "" && notificationVerifier == nil { var err error -<<<<<<< HEAD -======= ->>>>>>> Fix typos and formatting notificationVerifier, err = goverify.LoadPublicKeyFromFile(config.Global().PublicKeyPath) if err != nil { @@ -270,6 +267,6 @@ func handleDashboardZeroConfMessage(payload string) { if setHostname { config.SetGlobal(globalConf) - pubSubLog.Info("Hostname set with dashboard zeroconf signal") + pubSubLog.Info("Hostname set with dashboard zeroconf signal") } } From d546d900869f402d87972846c81753c90d194513 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 30 May 2018 09:36:11 +0300 Subject: [PATCH 35/54] Fix formatting --- coprocess_native.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coprocess_native.go b/coprocess_native.go index b0318de08dd..b4fa484ea61 100644 --- a/coprocess_native.go +++ b/coprocess_native.go @@ -60,7 +60,7 @@ func (c *CoProcessor) Dispatch(object *coprocess.Object) (*coprocess.Object, err newObjectPtr := (*C.struct_CoProcessMessage)(GlobalDispatcher.Dispatch(unsafe.Pointer(objectPtr))) defer C.free(unsafe.Pointer(newObjectPtr)) - if newObjectPtr == nil { + if newObjectPtr == nil { return nil, errors.New("Dispatch error") } From e7dc2bda33a61fc55d106bbf4419fb9358ef498a Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 1 Jun 2018 18:45:22 +0500 Subject: [PATCH 36/54] Fix whitelists failing when modify headers on same path V2 (#1749) * Fix whitelists Our rxPaths always built with the same order. First comes Whitelist, Ignore and Blacking, the rest of plugins after. So, if after we checked for whitelisting, we did not return early from the cycle, we can safely tell that white-listing failed. --- api_definition.go | 10 ++++++++++ api_definition_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ cert_go1.10_test.go | 2 -- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/api_definition.go b/api_definition.go index 9afc6bd1ba7..c3909b7e608 100644 --- a/api_definition.go +++ b/api_definition.go @@ -870,6 +870,7 @@ func (a *APISpec) URLAllowedAndIgnored(r *http.Request, rxPaths []URLSpec, white if !v.Spec.MatchString(strings.ToLower(r.URL.Path)) { continue } + if v.MethodActions != nil { // We are using an extended path set, check for the method methodMeta, matchMethodOk := v.MethodActions[r.Method] @@ -893,6 +894,15 @@ func (a *APISpec) URLAllowedAndIgnored(r *http.Request, rxPaths []URLSpec, white } } + if whiteListStatus { + // We have a whitelist, nothing gets through unless specifically defined + switch v.Status { + case WhiteList, BlackList, Ignored: + default: + return EndPointNotAllowed, nil + } + } + if v.TransformAction.Template != nil { return a.getURLStatus(v.Status), &v.TransformAction } diff --git a/api_definition_test.go b/api_definition_test.go index 1f02973613e..5aab2efa3ad 100644 --- a/api_definition_test.go +++ b/api_definition_test.go @@ -219,6 +219,48 @@ func TestIgnored(t *testing.T) { }) } +func TestWhitelistMethodWithAdditionalMiddleware(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + t.Run("Extended Paths", func(t *testing.T) { + buildAndLoadAPI(func(spec *APISpec) { + spec.UseKeylessAccess = true + spec.Proxy.ListenPath = "/" + + updateAPIVersion(spec, "v1", func(v *apidef.VersionInfo) { + v.UseExtendedPaths = true + + json.Unmarshal([]byte(`[ + { + "path": "/get", + "method_actions": {"GET": {"action": "no_action"}} + } + ]`), &v.ExtendedPaths.WhiteList) + json.Unmarshal([]byte(`[ + { + "add_headers": {"foo": "bar"}, + "path": "/get", + "method": "GET", + "act_on": false + } + ]`), &v.ExtendedPaths.TransformResponseHeader) + }) + spec.ResponseProcessors = []apidef.ResponseProcessor{{Name: "header_injector"}} + + }) + + //headers := map[string]string{"foo": "bar"} + ts.Run(t, []test.TestCase{ + + //Should get original upstream response + //{Method: "GET", Path: "/get", Code: 200, HeadersMatch: headers}, + //Reject not whitelisted (but know by upstream) path + {Method: "POST", Path: "/get", Code: 403}, + }...) + }) +} + func TestSyncAPISpecsDashboardSuccess(t *testing.T) { // Mock Dashboard ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/cert_go1.10_test.go b/cert_go1.10_test.go index 2c6c41e23f3..56b4564407f 100644 --- a/cert_go1.10_test.go +++ b/cert_go1.10_test.go @@ -200,9 +200,7 @@ func TestProxyTransport(t *testing.T) { buildAndLoadAPI(func(spec *APISpec) { spec.Proxy.ListenPath = "/" - spec.Proxy.TargetURL = upstream.URL spec.Proxy.Transport.SSLCipherSuites = []string{"TLS_RSA_WITH_AES_128_CBC_SHA"} - // Invalid proxy spec.Proxy.Transport.ProxyURL = proxy.URL }) From 8a812c933eefb1bfab12f9b5d0763221a2a0e8c7 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 2 Jun 2018 08:25:49 +0500 Subject: [PATCH 37/54] Fix body transform when input is JSON `json.Unmarshal` needs pointer to map --- mw_transform.go | 2 +- mw_transform_test.go | 44 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/mw_transform.go b/mw_transform.go index 8a1b67414fe..df8adeb3a88 100644 --- a/mw_transform.go +++ b/mw_transform.go @@ -76,7 +76,7 @@ func transformBody(r *http.Request, tmeta *TransformSpec, contextVars bool) erro return fmt.Errorf("error unmarshalling XML: %v", err) } case apidef.RequestJSON: - if err := json.Unmarshal(body, bodyData); err != nil { + if err := json.Unmarshal(body, &bodyData); err != nil { return err } default: diff --git a/mw_transform_test.go b/mw_transform_test.go index 93ae690fb5f..6f0a93af650 100644 --- a/mw_transform_test.go +++ b/mw_transform_test.go @@ -64,20 +64,46 @@ func TestTransformXMLCrash(t *testing.T) { } } -func testPrepareTransformJSONMarshal(tb testing.TB) (*TransformSpec, string) { - in := ` - Foo"oo - Bàr -` +func testPrepareTransformJSONMarshal(tb testing.TB, inputType string) (tmeta *TransformSpec, in string) { + tmeta = &TransformSpec{} tmpl := `[{{range $x, $s := .names.name}}{{$s | jsonMarshal}}{{if not $x}}, {{end}}{{end}}]` - tmeta := &TransformSpec{} tmeta.TemplateData.Input = apidef.RequestXML tmeta.Template = template.Must(apiTemplate.New("").Parse(tmpl)) + + switch inputType { + case "json": + tmeta.TemplateData.Input = apidef.RequestJSON + in = `{"names": { "name": ["Foo\"oo", "Bàr"] }}` + case "xml": + tmeta.TemplateData.Input = apidef.RequestXML + in = ` + Foo"oo + Bàr +` + } + return tmeta, in } -func TestTransformJSONMarshal(t *testing.T) { - tmeta, in := testPrepareTransformJSONMarshal(t) +func TestTransformJSONMarshalXMLInput(t *testing.T) { + tmeta, in := testPrepareTransformJSONMarshal(t, "xml") + + want := `["Foo\"oo", "Bàr"]` + r := testReq(t, "GET", "/", in) + if err := transformBody(r, tmeta, false); err != nil { + t.Fatalf("wanted nil error, got %v", err) + } + gotBs, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + if got := string(gotBs); got != want { + t.Fatalf("wanted body %q, got %q", want, got) + } +} + +func TestTransformJSONMarshalJSONInput(t *testing.T) { + tmeta, in := testPrepareTransformJSONMarshal(t, "json") want := `["Foo\"oo", "Bàr"]` r := testReq(t, "GET", "/", in) @@ -96,7 +122,7 @@ func TestTransformJSONMarshal(t *testing.T) { func BenchmarkTransformJSONMarshal(b *testing.B) { b.ReportAllocs() - tmeta, in := testPrepareTransformJSONMarshal(b) + tmeta, in := testPrepareTransformJSONMarshal(b, "xml") for i := 0; i < b.N; i++ { r := testReq(b, "GET", "/", in) From aa05fe2deeb15a4e69cda89b505d78a283baefb9 Mon Sep 17 00:00:00 2001 From: Ahmet Soormally Date: Sun, 3 Jun 2018 00:47:33 +0100 Subject: [PATCH 38/54] if json req/res is array, unmarshal to map[array] Better handling of arrays in both request & response body transforms. Places the array in map[array] so that we can unmarshal into map[string]interface{} --- mw_transform.go | 11 ++++++++++- mw_transform_test.go | 29 +++++++++++++++++++++++++++++ res_handler_transform.go | 14 +++++++++++--- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/mw_transform.go b/mw_transform.go index df8adeb3a88..a00aef6021b 100644 --- a/mw_transform.go +++ b/mw_transform.go @@ -76,9 +76,18 @@ func transformBody(r *http.Request, tmeta *TransformSpec, contextVars bool) erro return fmt.Errorf("error unmarshalling XML: %v", err) } case apidef.RequestJSON: - if err := json.Unmarshal(body, &bodyData); err != nil { + var tempBody interface{} + if err := json.Unmarshal(body, &tempBody); err != nil { return err } + + switch tempBody.(type) { + case []interface{}: + bodyData["array"] = tempBody + case map[string]interface{}: + bodyData = tempBody.(map[string]interface{}) + } + default: return fmt.Errorf("unsupported request input type: %v", tmeta.TemplateData.Input) } diff --git a/mw_transform_test.go b/mw_transform_test.go index 6f0a93af650..b4f8d904c12 100644 --- a/mw_transform_test.go +++ b/mw_transform_test.go @@ -119,6 +119,35 @@ func TestTransformJSONMarshalJSONInput(t *testing.T) { } } +func testPrepareTransformJSONMarshalArray(tb testing.TB) (tmeta *TransformSpec, in string) { + tmeta = &TransformSpec{} + tmpl := `[{{ range $key, $value := .array }}{{ if $key }},{{ end }}{{ .abc }}{{ end }}]` + tmeta.TemplateData.Input = apidef.RequestXML + tmeta.Template = template.Must(apiTemplate.New("").Parse(tmpl)) + + tmeta.TemplateData.Input = apidef.RequestJSON + in = `[{"abc": 123}, {"abc": 456}]` + + return tmeta, in +} + +func TestTransformJSONMarshalJSONArrayInput(t *testing.T) { + tmeta, in := testPrepareTransformJSONMarshalArray(t) + + want := `[123,456]` + r := testReq(t, "GET", "/", in) + if err := transformBody(r, tmeta, false); err != nil { + t.Fatalf("wanted nil error, got %v", err) + } + gotBs, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + if got := string(gotBs); got != want { + t.Fatalf("wanted body %q, got %q", want, got) + } +} + func BenchmarkTransformJSONMarshal(b *testing.B) { b.ReportAllocs() diff --git a/res_handler_transform.go b/res_handler_transform.go index 7e3ceb9d329..51134c0347d 100644 --- a/res_handler_transform.go +++ b/res_handler_transform.go @@ -85,7 +85,7 @@ func (h *ResponseTransformMiddleware) HandleResponse(rw http.ResponseWriter, res defer respBody.Close() // Put into an interface: - var bodyData interface{} + bodyData := make(map[string]interface{}) switch tmeta.TemplateData.Input { case apidef.RequestXML: mxj.XmlCharsetReader = WrappedCharsetReader @@ -102,10 +102,18 @@ func (h *ResponseTransformMiddleware) HandleResponse(rw http.ResponseWriter, res if err != nil { logger.WithError(err).Error("Error unmarshalling XML") } - default: // apidef.RequestJSON - if err := json.NewDecoder(respBody).Decode(&bodyData); err != nil { + default: + var tempBody interface{} + if err := json.NewDecoder(respBody).Decode(&tempBody); err != nil { logger.WithError(err).Error("Error unmarshalling JSON") } + + switch tempBody.(type) { + case []interface{}: + bodyData["array"] = tempBody + case map[string]interface{}: + bodyData = tempBody.(map[string]interface{}) + } } // Apply to template From 9d9d26012bc34b67c0430de1ee1faaad58947e86 Mon Sep 17 00:00:00 2001 From: Ahmet Soormally Date: Wed, 30 May 2018 21:15:02 +0100 Subject: [PATCH 39/54] switch rate limiter order so that keys are rate limited first Fixes #1755 This ensures that if a token goes over it's rate-limit, it does not count towards the global / api-level rate-limit. --- api_loader.go | 2 +- gateway_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ mw_auth_key_test.go | 1 - 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/api_loader.go b/api_loader.go index a4c927db0e7..243f6df7cb3 100644 --- a/api_loader.go +++ b/api_loader.go @@ -406,8 +406,8 @@ func processSpec(spec *APISpec, apisByListen map[string]int, mwAppendEnabled(&chainArray, &StripAuth{baseMid}) mwAppendEnabled(&chainArray, &KeyExpired{baseMid}) mwAppendEnabled(&chainArray, &AccessRightsCheck{baseMid}) - mwAppendEnabled(&chainArray, &RateLimitForAPI{BaseMiddleware: baseMid}) mwAppendEnabled(&chainArray, &RateLimitAndQuotaCheck{baseMid}) + mwAppendEnabled(&chainArray, &RateLimitForAPI{BaseMiddleware: baseMid}) mwAppendEnabled(&chainArray, &GranularAccessMiddleware{baseMid}) mwAppendEnabled(&chainArray, &ValidateJSON{BaseMiddleware: baseMid}) mwAppendEnabled(&chainArray, &TransformMiddleware{baseMid}) diff --git a/gateway_test.go b/gateway_test.go index e8d6d5369d7..d868ce1e31f 100644 --- a/gateway_test.go +++ b/gateway_test.go @@ -1218,3 +1218,47 @@ func TestKeepAliveConns(t *testing.T) { ts.Run(t, test.TestCase{Code: 200}) }) } + +// TestRateLimitForAPIAndRateLimitAndQuotaCheck ensures that the Rate Limit for the key is applied before the rate limit +// for the API. Meaning that a single token cannot reduce service availability for other tokens by simply going over the +// API's global rate limit. +func TestRateLimitForAPIAndRateLimitAndQuotaCheck(t *testing.T) { + + globalCfg := config.Global() + globalCfg.EnableNonTransactionalRateLimiter = false + globalCfg.EnableSentinelRateLImiter = true + config.SetGlobal(globalCfg) + + defer resetTestConfig() + + ts := newTykTestServer() + defer ts.Close() + + buildAndLoadAPI(func(spec *APISpec) { + spec.UseKeylessAccess = false + spec.DisableRateLimit = false + spec.OrgID = "default" + spec.GlobalRateLimit = apidef.GlobalRateLimit{ + Per: 60, + Rate: 2, + } + spec.Proxy.ListenPath = "/" + }) + + sess1token := createSession(func(s *user.SessionState) { + s.Rate = 1 + s.Per = 60 + }) + + sess2token := createSession(func(s *user.SessionState) { + s.Rate = 1 + s.Per = 60 + }) + + ts.Run(t, []test.TestCase{ + {Headers: map[string]string{"Authorization": sess1token}, Code: http.StatusOK, Path: "/"}, + {Headers: map[string]string{"Authorization": sess1token}, Code: http.StatusTooManyRequests, Path: "/"}, + {Headers: map[string]string{"Authorization": sess2token}, Code: http.StatusOK, Path: "/"}, + {Headers: map[string]string{"Authorization": sess2token}, Code: http.StatusTooManyRequests, Path: "/"}, + }...) +} diff --git a/mw_auth_key_test.go b/mw_auth_key_test.go index 4a439f48286..804c74f11bf 100644 --- a/mw_auth_key_test.go +++ b/mw_auth_key_test.go @@ -9,7 +9,6 @@ import ( "time" "github.com/justinas/alice" - "github.com/lonelycode/go-uuid/uuid" "github.com/TykTechnologies/tyk/user" From 424a51bf5ccf8a284eca8c9f5cc82dba71d5c725 Mon Sep 17 00:00:00 2001 From: Ahmet Soormally Date: Thu, 31 May 2018 13:45:43 +0100 Subject: [PATCH 40/54] add tyk_context to response body transform When `EnableContextVars: true`, request context data is added to `bodyData["_tyk_context"]` for use within response body transform templates. Similarly, we have also included `bodyData["_tyk_meta"]`. --- mw_transform.go | 1 - mw_url_rewrite.go | 7 +++--- res_handler_transform.go | 11 ++++++++- res_handler_transform_test.go | 43 +++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/mw_transform.go b/mw_transform.go index a00aef6021b..ea1e0a2b4a0 100644 --- a/mw_transform.go +++ b/mw_transform.go @@ -87,7 +87,6 @@ func transformBody(r *http.Request, tmeta *TransformSpec, contextVars bool) erro case map[string]interface{}: bodyData = tempBody.(map[string]interface{}) } - default: return fmt.Errorf("unsupported request input type: %v", tmeta.TemplateData.Input) } diff --git a/mw_url_rewrite.go b/mw_url_rewrite.go index 0bebd02af55..ad357f0f856 100644 --- a/mw_url_rewrite.go +++ b/mw_url_rewrite.go @@ -1,17 +1,16 @@ package main import ( + "fmt" + "io/ioutil" "net/http" + "net/textproto" "net/url" "reflect" "regexp" "strconv" "strings" - "fmt" - "io/ioutil" - "net/textproto" - "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/user" ) diff --git a/res_handler_transform.go b/res_handler_transform.go index 51134c0347d..8210df43ab3 100644 --- a/res_handler_transform.go +++ b/res_handler_transform.go @@ -102,7 +102,7 @@ func (h *ResponseTransformMiddleware) HandleResponse(rw http.ResponseWriter, res if err != nil { logger.WithError(err).Error("Error unmarshalling XML") } - default: + default: // apidef.RequestJSON var tempBody interface{} if err := json.NewDecoder(respBody).Decode(&tempBody); err != nil { logger.WithError(err).Error("Error unmarshalling JSON") @@ -116,6 +116,15 @@ func (h *ResponseTransformMiddleware) HandleResponse(rw http.ResponseWriter, res } } + if h.Spec.EnableContextVars { + bodyData["_tyk_context"] = ctxGetData(req) + } + + if tmeta.TemplateData.EnableSession { + session := ctxGetSession(req) + bodyData["_tyk_meta"] = session.MetaData + } + // Apply to template var bodyBuffer bytes.Buffer if err := tmeta.Template.Execute(&bodyBuffer, bodyData); err != nil { diff --git a/res_handler_transform_test.go b/res_handler_transform_test.go index a014f8bf161..d735fa408a1 100644 --- a/res_handler_transform_test.go +++ b/res_handler_transform_test.go @@ -84,3 +84,46 @@ func TestTransformResponseWithURLRewrite(t *testing.T) { }) }) } + +func TestTransformResponse_ContextVars(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + transformResponseConf := apidef.TemplateMeta{ + Path: "get", + Method: "GET", + TemplateData: apidef.TemplateData{ + Mode: "blob", + TemplateSource: base64.StdEncoding.EncodeToString([]byte(`{"foo":"{{._tyk_context.headers_Foo}}"}`)), + }, + } + + responseProcessorConf := []apidef.ResponseProcessor{{Name: "response_body_transform"}} + + // When Context Vars are disabled + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.ResponseProcessors = responseProcessorConf + updateAPIVersion(spec, "v1", func(v *apidef.VersionInfo) { + v.ExtendedPaths.TransformResponse = []apidef.TemplateMeta{transformResponseConf} + }) + }) + + ts.Run(t, test.TestCase{ + Headers: map[string]string{"Foo": "Bar"}, Path: "/get", Code: 200, BodyMatch: `{"foo":""}`, + }) + + // When Context Vars are enabled + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + spec.EnableContextVars = true + spec.ResponseProcessors = responseProcessorConf + updateAPIVersion(spec, "v1", func(v *apidef.VersionInfo) { + v.ExtendedPaths.TransformResponse = []apidef.TemplateMeta{transformResponseConf} + }) + }) + + ts.Run(t, test.TestCase{ + Headers: map[string]string{"Foo": "Bar"}, Path: "/get", Code: 200, BodyMatch: `{"foo":"Bar"}`, + }) +} From e887423ba9276d3c499a3fd4447062f3c758606f Mon Sep 17 00:00:00 2001 From: dencoded <33698537+dencoded@users.noreply.github.com> Date: Wed, 6 Jun 2018 13:36:45 -0400 Subject: [PATCH 41/54] NormalisePath and RegExp optimizations (#1689) changes for https://github.com/TykTechnologies/tyk/issues/1633 benchmark before: ``` time="May 8 14:09:43" level=info msg="Disabled feature" goos: darwin goarch: amd64 pkg: github.com/TykTechnologies/tyk BenchmarkURLReplacer-8 500000 17705 ns/op 848 B/op 44 allocs/op PASS ok github.com/TykTechnologies/tyk 9.332s ``` benchmark after: ``` time="May 9 12:28:00" level=info msg="Disabled feature" goos: darwin goarch: amd64 pkg: github.com/TykTechnologies/tyk BenchmarkURLReplacer-8 3000000 2124 ns/op 960 B/op 8 allocs/op PASS ok github.com/TykTechnologies/tyk 8.614s ``` NOTE: x6 throughput increase (!) benchcmp output (vs master): ``` benchmark old ns/op new ns/op delta BenchmarkURLReplacer-8 17705 2124 -88.00% benchmark old allocs new allocs delta BenchmarkURLReplacer-8 44 8 -81.82% benchmark old bytes new bytes delta BenchmarkURLReplacer-8 848 960 +13.21% ``` what was changed: - if config is already available in calling part - pass config to function - reduces fight for mutex (gives ~13% improvement) - added new package "`github.com/TykTechnologies/tyk/regexp`" (real performance boost comes from this change) which provides the same API as standard `"regexp"` but with caching inside (not for all functions - we can extend it as we go with our optimization changes for other tickets). Also, added some tests for it, still need to figure out how to test cache hits for `Regexp`'s methods --- analytics.go | 14 +- analytics_test.go | 24 +- api_definition.go | 2 +- apidef/api_definitions.go | 2 +- config/config.go | 4 +- coprocess_id_extractor.go | 2 +- event_system_test.go | 2 + handler_error.go | 2 +- handler_success.go | 2 +- lint/schema.go | 6 + main.go | 2 + mw_granular_access.go | 3 +- mw_hmac.go | 2 +- mw_url_rewrite.go | 2 +- regexp/cache.go | 88 +++ regexp/cache_regexp.go | 59 ++ regexp/cache_regexp_byte_ret_bool.go | 43 ++ regexp/cache_regexp_str_func_ret_str.go | 48 ++ ...ache_regexp_str_int_ret_slice_slice_str.go | 48 ++ regexp/cache_regexp_str_int_ret_slice_str.go | 48 ++ regexp/cache_regexp_str_ret_bool.go | 43 ++ regexp/cache_regexp_str_str_ret_str.go | 47 ++ regexp/regexp.go | 421 +++++++++++++ regexp/regexp_test.go | 569 ++++++++++++++++++ reverse_proxy.go | 2 +- 25 files changed, 1456 insertions(+), 29 deletions(-) create mode 100644 regexp/cache.go create mode 100644 regexp/cache_regexp.go create mode 100644 regexp/cache_regexp_byte_ret_bool.go create mode 100644 regexp/cache_regexp_str_func_ret_str.go create mode 100644 regexp/cache_regexp_str_int_ret_slice_slice_str.go create mode 100644 regexp/cache_regexp_str_int_ret_slice_str.go create mode 100644 regexp/cache_regexp_str_ret_bool.go create mode 100644 regexp/cache_regexp_str_str_ret_str.go create mode 100644 regexp/regexp.go create mode 100644 regexp/regexp_test.go diff --git a/analytics.go b/analytics.go index c0087f33e5d..07d72ac91b8 100644 --- a/analytics.go +++ b/analytics.go @@ -3,7 +3,6 @@ package main import ( "fmt" "net" - "regexp" "time" "github.com/jeffail/tunny" @@ -11,6 +10,7 @@ import ( "gopkg.in/vmihailenco/msgpack.v2" "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/regexp" "github.com/TykTechnologies/tyk/storage" ) @@ -118,14 +118,14 @@ func initNormalisationPatterns() (pats config.NormaliseURLPatterns) { return } -func (a *AnalyticsRecord) NormalisePath() { - if config.Global().AnalyticsConfig.NormaliseUrls.NormaliseUUIDs { - a.Path = config.Global().AnalyticsConfig.NormaliseUrls.CompiledPatternSet.UUIDs.ReplaceAllString(a.Path, "{uuid}") +func (a *AnalyticsRecord) NormalisePath(globalConfig *config.Config) { + if globalConfig.AnalyticsConfig.NormaliseUrls.NormaliseUUIDs { + a.Path = globalConfig.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.UUIDs.ReplaceAllString(a.Path, "{uuid}") } - if config.Global().AnalyticsConfig.NormaliseUrls.NormaliseNumbers { - a.Path = config.Global().AnalyticsConfig.NormaliseUrls.CompiledPatternSet.IDs.ReplaceAllString(a.Path, "/{id}") + if globalConfig.AnalyticsConfig.NormaliseUrls.NormaliseNumbers { + a.Path = globalConfig.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.IDs.ReplaceAllString(a.Path, "/{id}") } - for _, r := range config.Global().AnalyticsConfig.NormaliseUrls.CompiledPatternSet.Custom { + for _, r := range globalConfig.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.Custom { a.Path = r.ReplaceAllString(a.Path, "{var}") } } diff --git a/analytics_test.go b/analytics_test.go index 7dc77835626..07e3aa1445d 100644 --- a/analytics_test.go +++ b/analytics_test.go @@ -45,12 +45,12 @@ func TestURLReplacer(t *testing.T) { globalConf.AnalyticsConfig.NormaliseUrls.CompiledPatternSet = initNormalisationPatterns() config.SetGlobal(globalConf) - recordUUID1.NormalisePath() - recordUUID2.NormalisePath() - recordUUID3.NormalisePath() - recordUUID4.NormalisePath() - recordID1.NormalisePath() - recordCust.NormalisePath() + recordUUID1.NormalisePath(&globalConf) + recordUUID2.NormalisePath(&globalConf) + recordUUID3.NormalisePath(&globalConf) + recordUUID4.NormalisePath(&globalConf) + recordID1.NormalisePath(&globalConf) + recordCust.NormalisePath(&globalConf) if recordUUID1.Path != "/{uuid}/search" { t.Error("Path not altered, is:") @@ -105,12 +105,12 @@ func BenchmarkURLReplacer(b *testing.B) { recordID1 := AnalyticsRecord{Path: "/widgets/123456/getParams"} recordCust := AnalyticsRecord{Path: "/widgets/123456/getParams/ihatethisstring"} - recordUUID1.NormalisePath() - recordUUID2.NormalisePath() - recordUUID3.NormalisePath() - recordUUID4.NormalisePath() - recordID1.NormalisePath() - recordCust.NormalisePath() + recordUUID1.NormalisePath(&globalConf) + recordUUID2.NormalisePath(&globalConf) + recordUUID3.NormalisePath(&globalConf) + recordUUID4.NormalisePath(&globalConf) + recordID1.NormalisePath(&globalConf) + recordCust.NormalisePath(&globalConf) } } diff --git a/api_definition.go b/api_definition.go index c3909b7e608..0a020b128f2 100644 --- a/api_definition.go +++ b/api_definition.go @@ -11,7 +11,6 @@ import ( "net/url" "os" "path/filepath" - "regexp" "strings" "sync" "sync/atomic" @@ -23,6 +22,7 @@ import ( "github.com/TykTechnologies/gojsonschema" "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/regexp" "github.com/TykTechnologies/tyk/storage" ) diff --git a/apidef/api_definitions.go b/apidef/api_definitions.go index d87f7d59cae..faee754576c 100644 --- a/apidef/api_definitions.go +++ b/apidef/api_definitions.go @@ -3,12 +3,12 @@ package apidef import ( "encoding/base64" "encoding/json" - "regexp" "github.com/lonelycode/osin" "gopkg.in/mgo.v2/bson" "github.com/TykTechnologies/gojsonschema" + "github.com/TykTechnologies/tyk/regexp" ) type AuthProviderCode string diff --git a/config/config.go b/config/config.go index f5a22950c23..0ca116c0ad5 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,6 @@ import ( "io" "io/ioutil" "os" - "regexp" "sync" "sync/atomic" @@ -14,6 +13,7 @@ import ( "github.com/TykTechnologies/tyk/apidef" logger "github.com/TykTechnologies/tyk/log" + "github.com/TykTechnologies/tyk/regexp" ) var log = logger.Get() @@ -287,6 +287,8 @@ type Config struct { VersionHeader string `json:"version_header"` EnableHashedKeysListing bool `json:"enable_hashed_keys_listing"` MinTokenLength int `json:"min_token_length"` + DisableRegexpCache bool `json:"disable_regexp_cache"` + RegexpCacheExpire int32 `json:"regexp_cache_expire"` } type CertData struct { diff --git a/coprocess_id_extractor.go b/coprocess_id_extractor.go index 7be112d878b..03e2347487b 100644 --- a/coprocess_id_extractor.go +++ b/coprocess_id_extractor.go @@ -6,7 +6,6 @@ import ( "fmt" "io/ioutil" "net/http" - "regexp" "strings" "time" @@ -15,6 +14,7 @@ import ( "gopkg.in/xmlpath.v2" "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/regexp" "github.com/TykTechnologies/tyk/user" ) diff --git a/event_system_test.go b/event_system_test.go index 24d9885a12c..3dcf883eb48 100644 --- a/event_system_test.go +++ b/event_system_test.go @@ -157,6 +157,8 @@ func TestInitGenericEventHandlers(t *testing.T) { } func BenchmarkInitGenericEventHandlers(b *testing.B) { + b.ReportAllocs() + conf := prepareEventsConf() for i := 0; i < b.N; i++ { initGenericEventHandlers(conf) diff --git a/handler_error.go b/handler_error.go index 382dc24348a..463983e117b 100644 --- a/handler_error.go +++ b/handler_error.go @@ -176,7 +176,7 @@ func (e *ErrorHandler) HandleError(w http.ResponseWriter, r *http.Request, errMs record.SetExpiry(expiresAfter) if e.Spec.GlobalConfig.AnalyticsConfig.NormaliseUrls.Enabled { - record.NormalisePath() + record.NormalisePath(&e.Spec.GlobalConfig) } go analytics.RecordHit(record) diff --git a/handler_success.go b/handler_success.go index 6287d6fee25..5ddbdd80a7c 100644 --- a/handler_success.go +++ b/handler_success.go @@ -188,7 +188,7 @@ func (s *SuccessHandler) RecordHit(r *http.Request, timing int64, code int, requ record.SetExpiry(expiresAfter) if s.Spec.GlobalConfig.AnalyticsConfig.NormaliseUrls.Enabled { - record.NormalisePath() + record.NormalisePath(&s.Spec.GlobalConfig) } go analytics.RecordHit(record) diff --git a/lint/schema.go b/lint/schema.go index 35dff81610d..13df8fd9091 100644 --- a/lint/schema.go +++ b/lint/schema.go @@ -733,6 +733,12 @@ const confSchema = `{ }, "min_token_length": { "type": "integer" + }, + "disable_regexp_cache": { + "type": "boolean" + }, + "regexp_cache_expire": { + "type": "integer" } } }` diff --git a/main.go b/main.go index beb15a27e83..c16a2e68a24 100644 --- a/main.go +++ b/main.go @@ -44,6 +44,7 @@ import ( "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/lint" logger "github.com/TykTechnologies/tyk/log" + "github.com/TykTechnologies/tyk/regexp" "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/user" ) @@ -894,6 +895,7 @@ func afterConfSetup(conf *config.Config) { GlobalRPCPingTimeout = time.Second * time.Duration(conf.SlaveOptions.PingTimeout) GlobalRPCCallTimeout = time.Second * time.Duration(conf.SlaveOptions.CallTimeout) initGenericEventHandlers(conf) + regexp.ResetCache(time.Second*time.Duration(conf.RegexpCacheExpire), !conf.DisableRegexpCache) } var hostDetails struct { diff --git a/mw_granular_access.go b/mw_granular_access.go index ed2be3b2292..a54bab9394f 100644 --- a/mw_granular_access.go +++ b/mw_granular_access.go @@ -3,7 +3,8 @@ package main import ( "errors" "net/http" - "regexp" + + "github.com/TykTechnologies/tyk/regexp" ) // GranularAccessMiddleware will check if a URL is specifically enabled for the key diff --git a/mw_hmac.go b/mw_hmac.go index ef9c759de9b..3dbe124ea4a 100644 --- a/mw_hmac.go +++ b/mw_hmac.go @@ -8,13 +8,13 @@ import ( "math" "net/http" "net/url" - "regexp" "strings" "time" "github.com/Sirupsen/logrus" "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/regexp" "github.com/TykTechnologies/tyk/user" ) diff --git a/mw_url_rewrite.go b/mw_url_rewrite.go index ad357f0f856..114ada99b97 100644 --- a/mw_url_rewrite.go +++ b/mw_url_rewrite.go @@ -7,11 +7,11 @@ import ( "net/textproto" "net/url" "reflect" - "regexp" "strconv" "strings" "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/regexp" "github.com/TykTechnologies/tyk/user" ) diff --git a/regexp/cache.go b/regexp/cache.go new file mode 100644 index 00000000000..d8a6b450d8e --- /dev/null +++ b/regexp/cache.go @@ -0,0 +1,88 @@ +package regexp + +import ( + "regexp" + "time" + + gocache "github.com/pmylund/go-cache" +) + +const ( + defaultCacheItemTTL = 60 * time.Second + defaultCacheCleanupInterval = 5 * time.Minute + + maxKeySize = 1024 + maxValueSize = 2048 +) + +type cache struct { + *gocache.Cache + isEnabled bool + ttl time.Duration +} + +func newCache(ttl time.Duration, isEnabled bool) *cache { + return &cache{ + Cache: gocache.New(ttl, defaultCacheCleanupInterval), + isEnabled: isEnabled, + ttl: ttl, + } +} + +func (c *cache) enabled() bool { + return c.isEnabled && c.Cache != nil +} + +func (c *cache) add(key string, value interface{}) { + c.Add(key, value, c.ttl) +} + +func (c *cache) getRegexp(key string) (*regexp.Regexp, bool) { + if val, found := c.Get(key); found { + return val.(*regexp.Regexp).Copy(), true + } + + return nil, false +} + +func (c *cache) getString(key string) (string, bool) { + if val, found := c.Get(key); found { + return val.(string), true + } + + return "", false +} + +func (c *cache) getStrSlice(key string) ([]string, bool) { + if val, found := c.Get(key); found { + return val.([]string), true + } + + return []string{}, false +} + +func (c *cache) getStrSliceOfSlices(key string) ([][]string, bool) { + if val, found := c.Get(key); found { + return val.([][]string), true + } + + return [][]string{}, false +} + +func (c *cache) getBool(key string) (bool, bool) { + if val, found := c.Get(key); found { + return val.(bool), true + } + + return false, false +} + +func (c *cache) reset(ttl time.Duration, isEnabled bool) { + if c.Cache == nil { + return + } + + c.isEnabled = isEnabled + c.ttl = ttl + c.Flush() +} diff --git a/regexp/cache_regexp.go b/regexp/cache_regexp.go new file mode 100644 index 00000000000..dc1f53029f9 --- /dev/null +++ b/regexp/cache_regexp.go @@ -0,0 +1,59 @@ +package regexp + +import ( + "regexp" + "time" +) + +type regexpCache struct { + *cache + noCacheFunc func(string) (*regexp.Regexp, error) +} + +func newRegexpCache(ttl time.Duration, isEnabled bool, fn func(string) (*regexp.Regexp, error)) *regexpCache { + return ®expCache{ + cache: newCache( + ttl, + isEnabled, + ), + noCacheFunc: fn, + } +} + +func (c *regexpCache) doNoCacheFunc(str string) (*Regexp, error) { + rx, err := c.noCacheFunc(str) + if err != nil { + return nil, err + } + + return &Regexp{ + rx, + false, + }, + nil +} + +func (c *regexpCache) do(str string) (*Regexp, error) { + // return if cache is not enabled + if !c.enabled() { + return c.doNoCacheFunc(str) + } + + // cache hit + if rx, found := c.getRegexp(str); found { + return &Regexp{ + rx, + true, + }, + nil + } + + // cache miss, add to cache + regExp, err := c.doNoCacheFunc(str) + if err != nil { + return nil, err + } + c.add(str, regExp.Regexp) + + return regExp, nil +} diff --git a/regexp/cache_regexp_byte_ret_bool.go b/regexp/cache_regexp_byte_ret_bool.go new file mode 100644 index 00000000000..f3ccbc281f9 --- /dev/null +++ b/regexp/cache_regexp_byte_ret_bool.go @@ -0,0 +1,43 @@ +package regexp + +import ( + "regexp" + "time" +) + +type regexpByteRetBoolCache struct { + *cache +} + +func newRegexpByteRetBoolCache(ttl time.Duration, isEnabled bool) *regexpByteRetBoolCache { + return ®expByteRetBoolCache{ + cache: newCache( + ttl, + isEnabled, + ), + } +} + +func (c *regexpByteRetBoolCache) do(r *regexp.Regexp, b []byte, noCacheFn func([]byte) bool) bool { + // return if cache is not enabled + if !c.enabled() { + return noCacheFn(b) + } + + // generate key, check key size + key := r.String() + string(b) + if len(key) > maxKeySize { + return noCacheFn(b) + } + + // cache hit + if res, found := c.getBool(key); found { + return res + } + + // cache miss, add to cache + res := noCacheFn(b) + c.add(key, res) + + return res +} diff --git a/regexp/cache_regexp_str_func_ret_str.go b/regexp/cache_regexp_str_func_ret_str.go new file mode 100644 index 00000000000..8f99fa11c34 --- /dev/null +++ b/regexp/cache_regexp_str_func_ret_str.go @@ -0,0 +1,48 @@ +package regexp + +import ( + "fmt" + "regexp" + "time" +) + +type regexpStrFuncRetStrCache struct { + *cache +} + +func newRegexpStrFuncRetStrCache(ttl time.Duration, isEnabled bool) *regexpStrFuncRetStrCache { + return ®expStrFuncRetStrCache{ + cache: newCache( + ttl, + isEnabled, + ), + } +} + +func (c *regexpStrFuncRetStrCache) do(r *regexp.Regexp, src string, repl func(string) string, noCacheFn func(string, func(string) string) string) string { + // return if cache is not enabled + if !c.enabled() { + return noCacheFn(src, repl) + } + + // generate key, check key size + key := r.String() + src + fmt.Sprintf("%p", repl) + if len(key) > maxKeySize { + return noCacheFn(src, repl) + } + + // cache hit + if res, found := c.getString(key); found { + return res + } + + // cache miss, add to cache if value is not too big + res := noCacheFn(src, repl) + if len(res) > maxValueSize { + return res + } + + c.add(key, res) + + return res +} diff --git a/regexp/cache_regexp_str_int_ret_slice_slice_str.go b/regexp/cache_regexp_str_int_ret_slice_slice_str.go new file mode 100644 index 00000000000..84b24fb6097 --- /dev/null +++ b/regexp/cache_regexp_str_int_ret_slice_slice_str.go @@ -0,0 +1,48 @@ +package regexp + +import ( + "regexp" + "strconv" + "time" +) + +type regexpStrIntRetSliceSliceStrCache struct { + *cache +} + +func newRegexpStrIntRetSliceSliceStrCache(ttl time.Duration, isEnabled bool) *regexpStrIntRetSliceSliceStrCache { + return ®expStrIntRetSliceSliceStrCache{ + cache: newCache( + ttl, + isEnabled, + ), + } +} + +func (c *regexpStrIntRetSliceSliceStrCache) do(r *regexp.Regexp, s string, n int, noCacheFn func(s string, n int) [][]string) [][]string { + // return if cache is not enabled + if !c.enabled() { + return noCacheFn(s, n) + } + + // generate key, check key size + key := r.String() + s + strconv.Itoa(n) + if len(key) > maxKeySize { + return noCacheFn(s, n) + } + + // cache hit + if res, found := c.getStrSliceOfSlices(key); found { + return res + } + + // cache miss, add to cache if value is not too big + res := noCacheFn(s, n) + if len(res) > maxValueSize { + return res + } + + c.add(key, res) + + return res +} diff --git a/regexp/cache_regexp_str_int_ret_slice_str.go b/regexp/cache_regexp_str_int_ret_slice_str.go new file mode 100644 index 00000000000..d1cbd8942a8 --- /dev/null +++ b/regexp/cache_regexp_str_int_ret_slice_str.go @@ -0,0 +1,48 @@ +package regexp + +import ( + "regexp" + "strconv" + "time" +) + +type regexpStrIntRetSliceStrCache struct { + *cache +} + +func newRegexpStrIntRetSliceStrCache(ttl time.Duration, isEnabled bool) *regexpStrIntRetSliceStrCache { + return ®expStrIntRetSliceStrCache{ + cache: newCache( + ttl, + isEnabled, + ), + } +} + +func (c *regexpStrIntRetSliceStrCache) do(r *regexp.Regexp, s string, n int, noCacheFn func(s string, n int) []string) []string { + // return if cache is not enabled + if !c.enabled() { + return noCacheFn(s, n) + } + + // generate key, check key size + key := r.String() + s + strconv.Itoa(n) + if len(key) > maxKeySize { + return noCacheFn(s, n) + } + + // cache hit + if res, found := c.getStrSlice(key); found { + return res + } + + // cache miss, add to cache if value is not too big + res := noCacheFn(s, n) + if len(res) > maxValueSize { + return res + } + + c.add(key, res) + + return res +} diff --git a/regexp/cache_regexp_str_ret_bool.go b/regexp/cache_regexp_str_ret_bool.go new file mode 100644 index 00000000000..df87118ec69 --- /dev/null +++ b/regexp/cache_regexp_str_ret_bool.go @@ -0,0 +1,43 @@ +package regexp + +import ( + "regexp" + "time" +) + +type regexpStrRetBoolCache struct { + *cache +} + +func newRegexpStrRetBoolCache(ttl time.Duration, isEnabled bool) *regexpStrRetBoolCache { + return ®expStrRetBoolCache{ + cache: newCache( + ttl, + isEnabled, + ), + } +} + +func (c *regexpStrRetBoolCache) do(r *regexp.Regexp, s string, noCacheFn func(string) bool) bool { + // return if cache is not enabled + if !c.enabled() { + return noCacheFn(s) + } + + // generate key, check key size + key := r.String() + s + if len(key) > maxKeySize { + return noCacheFn(s) + } + + // cache hit + if res, found := c.getBool(key); found { + return res + } + + // cache miss, add to cache + res := noCacheFn(s) + c.add(key, res) + + return res +} diff --git a/regexp/cache_regexp_str_str_ret_str.go b/regexp/cache_regexp_str_str_ret_str.go new file mode 100644 index 00000000000..8ff0ea455ad --- /dev/null +++ b/regexp/cache_regexp_str_str_ret_str.go @@ -0,0 +1,47 @@ +package regexp + +import ( + "regexp" + "time" +) + +type regexpStrStrRetStrCache struct { + *cache +} + +func newRegexpStrStrRetStrCache(ttl time.Duration, isEnabled bool) *regexpStrStrRetStrCache { + return ®expStrStrRetStrCache{ + cache: newCache( + ttl, + isEnabled, + ), + } +} + +func (c *regexpStrStrRetStrCache) do(r *regexp.Regexp, src string, repl string, noCacheFn func(string, string) string) string { + // return if cache is not enabled + if !c.enabled() { + return noCacheFn(src, repl) + } + + // generate key, check key size + key := r.String() + src + repl + if len(key) > maxKeySize { + return noCacheFn(src, repl) + } + + // cache hit + if res, found := c.getString(key); found { + return res + } + + // cache miss, add to cache if value is not too big + res := noCacheFn(src, repl) + if len(res) > maxValueSize { + return res + } + + c.add(key, res) + + return res +} diff --git a/regexp/regexp.go b/regexp/regexp.go new file mode 100644 index 00000000000..105c5d66667 --- /dev/null +++ b/regexp/regexp.go @@ -0,0 +1,421 @@ +/* +Package regexp implmenets API of Go's native "regexp" but with caching results in memory +*/ +package regexp + +import ( + "io" + "regexp" + "strconv" + "time" +) + +var ( + compileCache = newRegexpCache(defaultCacheItemTTL, true, regexp.Compile) + compilePOSIXCache = newRegexpCache(defaultCacheItemTTL, true, regexp.CompilePOSIX) + matchStringCache = newRegexpStrRetBoolCache(defaultCacheItemTTL, true) + matchCache = newRegexpByteRetBoolCache(defaultCacheItemTTL, true) + replaceAllStringCache = newRegexpStrStrRetStrCache(defaultCacheItemTTL, true) + replaceAllLiteralStringCache = newRegexpStrStrRetStrCache(defaultCacheItemTTL, true) + replaceAllStringFuncCache = newRegexpStrFuncRetStrCache(defaultCacheItemTTL, true) + findAllStringCache = newRegexpStrIntRetSliceStrCache(defaultCacheItemTTL, true) + findAllStringSubmatchCache = newRegexpStrIntRetSliceSliceStrCache(defaultCacheItemTTL, true) +) + +// Regexp is a wrapper around regexp.Regexp but with caching +type Regexp struct { + *regexp.Regexp + FromCache bool +} + +// ResetCache resets cache to initial state +func ResetCache(ttl time.Duration, isEnabled bool) { + if ttl == 0 { + ttl = defaultCacheItemTTL + } + + compileCache.reset(ttl, isEnabled) + compilePOSIXCache.reset(ttl, isEnabled) + matchStringCache.reset(ttl, isEnabled) + matchCache.reset(ttl, isEnabled) + replaceAllStringCache.reset(ttl, isEnabled) + replaceAllLiteralStringCache.reset(ttl, isEnabled) + replaceAllStringFuncCache.reset(ttl, isEnabled) + findAllStringCache.reset(ttl, isEnabled) + findAllStringSubmatchCache.reset(ttl, isEnabled) +} + +// Compile does the same as regexp.Compile but returns cached *Regexp instead. +func Compile(expr string) (*Regexp, error) { + return compileCache.do(expr) +} + +// CompilePOSIX does the same as regexp.CompilePOSIX but returns cached *Regexp instead. +func CompilePOSIX(expr string) (*Regexp, error) { + return compilePOSIXCache.do(expr) +} + +// MustCompile is the same as regexp.MustCompile but returns cached *Regexp instead. +func MustCompile(str string) *Regexp { + regexp, err := Compile(str) + if err != nil { + panic(`regexp: Compile(` + quote(str) + `): ` + err.Error()) + } + return regexp +} + +// MustCompilePOSIX is the same as regexp.MustCompilePOSIX but returns cached *Regexp instead. +func MustCompilePOSIX(str string) *Regexp { + regexp, err := CompilePOSIX(str) + if err != nil { + panic(`regexp: CompilePOSIX(` + quote(str) + `): ` + err.Error()) + } + return regexp +} + +// MatchString is the same as regexp.MatchString but returns cached result instead. +func MatchString(pattern string, s string) (matched bool, err error) { + re, err := Compile(pattern) + if err != nil { + return false, err + } + return re.MatchString(s), nil +} + +// Match is the same as regexp.Match but returns cached result instead. +func Match(pattern string, b []byte) (matched bool, err error) { + re, err := Compile(pattern) + if err != nil { + return false, err + } + return re.Match(b), nil +} + +// QuoteMeta is the same as regexp.QuoteMeta but returns cached result instead. +func QuoteMeta(s string) string { + // TODO: add cache for QuoteMeta + return regexp.QuoteMeta(s) +} + +func quote(s string) string { + if strconv.CanBackquote(s) { + return "`" + s + "`" + } + return strconv.Quote(s) +} + +// String returns the source text used to compile the wrapped regular expression. +func (re *Regexp) String() string { + if re.Regexp == nil { + return "" + } + return re.Regexp.String() +} + +// Copy returns a new Regexp object copied from re. +// +// When using a Regexp in multiple goroutines, giving each goroutine +// its own copy helps to avoid lock contention. +func (re *Regexp) Copy() *Regexp { + reCopy := &Regexp{ + FromCache: re.FromCache, + } + if re.Regexp != nil { + reCopy.Regexp = re.Regexp.Copy() + } + return reCopy +} + +// Longest calls regexp.Regexp.Longest of wrapped regular expression +func (re *Regexp) Longest() { + if re.Regexp == nil { + re.Regexp.Longest() + } +} + +// NumSubexp returns result of regexp.Regexp.NumSubexp of wrapped regular expression. +func (re *Regexp) NumSubexp() int { + if re.Regexp == nil { + return 0 + } + return re.Regexp.NumSubexp() +} + +// SubexpNames returns result of regexp.Regexp.SubexpNames of wrapped regular expression. +func (re *Regexp) SubexpNames() []string { + if re.Regexp == nil { + return []string{} + } + return re.Regexp.SubexpNames() +} + +// LiteralPrefix returns a literal string that must begin any match +// Calls regexp.Regexp.LiteralPrefix +func (re *Regexp) LiteralPrefix() (prefix string, complete bool) { + if re.Regexp == nil { + return "", false + } + return re.Regexp.LiteralPrefix() +} + +// MatchReader reports whether the Regexp matches the text read by the +// RuneReader. +// Calls regexp.Regexp.MatchReader (NO CACHE) +func (re *Regexp) MatchReader(r io.RuneReader) bool { + if re.Regexp == nil { + return false + } + return re.Regexp.MatchReader(r) +} + +// MatchString reports whether the Regexp matches the string s. +func (re *Regexp) MatchString(s string) bool { + if re.Regexp == nil { + return false + } + return matchStringCache.do(re.Regexp, s, re.Regexp.MatchString) +} + +// Match reports whether the Regexp matches the byte slice b. +func (re *Regexp) Match(b []byte) bool { + if re.Regexp == nil { + return false + } + return matchCache.do(re.Regexp, b, re.Regexp.Match) +} + +// ReplaceAllString is the same as regexp.Regexp.ReplaceAllString but returns cached result instead. +func (re *Regexp) ReplaceAllString(src, repl string) string { + if re.Regexp == nil { + return "" + } + return replaceAllStringCache.do(re.Regexp, src, repl, re.Regexp.ReplaceAllString) +} + +// ReplaceAllLiteralString is the same as regexp.Regexp.ReplaceAllLiteralString but returns cached result instead. +func (re *Regexp) ReplaceAllLiteralString(src, repl string) string { + if re.Regexp == nil { + return "" + } + return replaceAllLiteralStringCache.do(re.Regexp, src, repl, re.Regexp.ReplaceAllLiteralString) +} + +// ReplaceAllStringFunc is the same as regexp.Regexp.ReplaceAllStringFunc but returns cached result instead. +func (re *Regexp) ReplaceAllStringFunc(src string, repl func(string) string) string { + if re.Regexp == nil { + return "" + } + return replaceAllStringFuncCache.do(re.Regexp, src, repl, re.Regexp.ReplaceAllStringFunc) +} + +// ReplaceAll is the same as regexp.Regexp.ReplaceAll but returns cached result instead. +func (re *Regexp) ReplaceAll(src, repl []byte) []byte { + if re.Regexp == nil { + return []byte{} + } + // TODO: add cache for ReplaceAll + return re.Regexp.ReplaceAll(src, repl) +} + +// ReplaceAllLiteral is the same as regexp.Regexp.ReplaceAllLiteral but returns cached result instead. +func (re *Regexp) ReplaceAllLiteral(src, repl []byte) []byte { + if re.Regexp == nil { + return []byte{} + } + // TODO: add cache for ReplaceAllLiteral + return re.Regexp.ReplaceAllLiteral(src, repl) +} + +// ReplaceAllFunc is the same as regexp.Regexp.ReplaceAllFunc but returns cached result instead. +func (re *Regexp) ReplaceAllFunc(src []byte, repl func([]byte) []byte) []byte { + if re.Regexp == nil { + return []byte{} + } + // TODO: add cache for ReplaceAllFunc + return re.Regexp.ReplaceAllFunc(src, repl) +} + +// Find is the same as regexp.Regexp.Find but returns cached result instead. +func (re *Regexp) Find(b []byte) []byte { + if re.Regexp == nil { + return []byte{} + } + // TODO: add cache for Find + return re.Regexp.Find(b) +} + +// FindIndex is the same as regexp.Regexp.FindIndex but returns cached result instead. +func (re *Regexp) FindIndex(b []byte) (loc []int) { + if re.Regexp == nil { + return + } + // TODO: add cache for FindIndex + return re.Regexp.FindIndex(b) +} + +// FindString is the same as regexp.Regexp.FindString but returns cached result instead. +func (re *Regexp) FindString(s string) string { + if re.Regexp == nil { + return "" + } + // TODO: add cache for FindString + return re.Regexp.FindString(s) +} + +// FindStringIndex is the same as regexp.Regexp.FindStringIndex but returns cached result instead. +func (re *Regexp) FindStringIndex(s string) (loc []int) { + if re.Regexp == nil { + return + } + // TODO: add cache for FindStringIndex + return re.Regexp.FindStringIndex(s) +} + +// FindReaderIndex is the same as regexp.Regexp.FindReaderIndex (NO CACHE). +func (re *Regexp) FindReaderIndex(r io.RuneReader) (loc []int) { + if re.Regexp == nil { + return + } + return re.Regexp.FindReaderIndex(r) +} + +// FindSubmatch is the same as regexp.Regexp.FindSubmatch but returns cached result instead. +func (re *Regexp) FindSubmatch(b []byte) [][]byte { + if re.Regexp == nil { + return [][]byte{} + } + // TODO: add cache for FindSubmatch + return re.Regexp.FindSubmatch(b) +} + +// Expand is the same as regexp.Regexp.Expand but returns cached result instead. +func (re *Regexp) Expand(dst []byte, template []byte, src []byte, match []int) []byte { + if re.Regexp == nil { + return []byte{} + } + // TODO: add cache for Expand + return re.Regexp.Expand(dst, template, src, match) +} + +// ExpandString is the same as regexp.Regexp.ExpandString but returns cached result instead. +func (re *Regexp) ExpandString(dst []byte, template string, src string, match []int) []byte { + if re.Regexp == nil { + return []byte{} + } + // TODO: add cache for ExpandString + return re.Regexp.ExpandString(dst, template, src, match) +} + +// FindSubmatchIndex is the same as regexp.Regexp.FindSubmatchIndex but returns cached result instead. +func (re *Regexp) FindSubmatchIndex(b []byte) []int { + if re.Regexp == nil { + return []int{} + } + // TODO: add cache for ExpandString + return re.Regexp.FindSubmatchIndex(b) +} + +// FindStringSubmatch is the same as regexp.Regexp.FindStringSubmatch but returns cached result instead. +func (re *Regexp) FindStringSubmatch(s string) []string { + if re.Regexp == nil { + return []string{} + } + // TODO: add cache for FindStringSubmatch + return re.Regexp.FindStringSubmatch(s) +} + +// FindStringSubmatchIndex is the same as regexp.Regexp.FindStringSubmatchIndex but returns cached result instead. +func (re *Regexp) FindStringSubmatchIndex(s string) []int { + if re.Regexp == nil { + return []int{} + } + // TODO: add cache for FindStringSubmatchIndex + return re.Regexp.FindStringSubmatchIndex(s) +} + +// FindReaderSubmatchIndex is the same as regexp.Regexp.FindReaderSubmatchIndex (NO CACHE). +func (re *Regexp) FindReaderSubmatchIndex(r io.RuneReader) []int { + if re.Regexp == nil { + return []int{} + } + return re.Regexp.FindReaderSubmatchIndex(r) +} + +// FindAll is the same as regexp.Regexp.FindAll but returns cached result instead. +func (re *Regexp) FindAll(b []byte, n int) [][]byte { + if re.Regexp == nil { + return [][]byte{} + } + // TODO: add cache for FindAll + return re.Regexp.FindAll(b, n) +} + +// FindAllIndex is the same as regexp.Regexp.FindAllIndex but returns cached result instead. +func (re *Regexp) FindAllIndex(b []byte, n int) [][]int { + if re.Regexp == nil { + return [][]int{} + } + // TODO: add cache for FindAllIndex + return re.Regexp.FindAllIndex(b, n) +} + +// FindAllString is the same as regexp.Regexp.FindAllString but returns cached result instead. +func (re *Regexp) FindAllString(s string, n int) []string { + if re.Regexp == nil { + return []string{} + } + return findAllStringCache.do(re.Regexp, s, n, re.Regexp.FindAllString) +} + +// FindAllStringIndex is the same as regexp.Regexp.FindAllStringIndex but returns cached result instead. +func (re *Regexp) FindAllStringIndex(s string, n int) [][]int { + if re.Regexp == nil { + return [][]int{} + } + // TODO: add cache for FindAllStringIndex + return re.Regexp.FindAllStringIndex(s, n) +} + +// FindAllSubmatch is the same as regexp.Regexp.FindAllSubmatch but returns cached result instead. +func (re *Regexp) FindAllSubmatch(b []byte, n int) [][][]byte { + if re.Regexp == nil { + return [][][]byte{} + } + // TODO: add cache for FindAllSubmatch + return re.Regexp.FindAllSubmatch(b, n) +} + +// FindAllSubmatchIndex is the same as regexp.Regexp.FindAllSubmatchIndex but returns cached result instead. +func (re *Regexp) FindAllSubmatchIndex(b []byte, n int) [][]int { + if re.Regexp == nil { + return [][]int{} + } + // TODO: add cache for FindAllSubmatchIndex + return re.Regexp.FindAllSubmatchIndex(b, n) +} + +// FindAllStringSubmatch is the same as regexp.Regexp.FindAllStringSubmatch but returns cached result instead. +func (re *Regexp) FindAllStringSubmatch(s string, n int) [][]string { + if re.Regexp == nil { + return [][]string{} + } + return findAllStringSubmatchCache.do(re.Regexp, s, n, re.Regexp.FindAllStringSubmatch) +} + +// FindAllStringSubmatchIndex is the same as regexp.Regexp.FindAllStringSubmatchIndex but returns cached result instead. +func (re *Regexp) FindAllStringSubmatchIndex(s string, n int) [][]int { + if re.Regexp == nil { + return [][]int{} + } + // TODO: add cache for FindAllStringSubmatchIndex + return re.Regexp.FindAllStringSubmatchIndex(s, n) +} + +// Split is the same as regexp.Regexp.Split but returns cached result instead. +func (re *Regexp) Split(s string, n int) []string { + if re.Regexp == nil { + return []string{} + } + // TODO: add cache for Split + return re.Regexp.Split(s, n) +} diff --git a/regexp/regexp_test.go b/regexp/regexp_test.go new file mode 100644 index 00000000000..b6590210b3b --- /dev/null +++ b/regexp/regexp_test.go @@ -0,0 +1,569 @@ +package regexp + +import ( + "reflect" + "strings" + "testing" +) + +func TestCompile(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + // 1st miss + rx, err := Compile("^abc.*$") + if err != nil { + t.Fatal(err) + } + if rx.FromCache { + t.Error("Regexp shoul not be from cache") + } + if !rx.MatchString("abcxyz") { + t.Error("String didn't match to compiled regexp: ", rx.String()) + } + // 2nd hit + rx2, err := Compile("^abc.*$") + if err != nil { + t.Fatal(err) + } + if !rx2.FromCache { + t.Error("Regexp shoul be from cache") + } + if !rx2.MatchString("abcxyz") { + t.Error("String didn't match to compiled regexp: ", rx2.String()) + } +} + +func BenchmarkRegExpCompile(b *testing.B) { + ResetCache(defaultCacheItemTTL, true) + + b.ReportAllocs() + + var rx *Regexp + var err error + + for i := 0; i < b.N; i++ { + rx, err = Compile("^abc.*$") + if err != nil { + b.Fatal(err) + } + } + + b.Log(rx) +} + +func TestCompilePOSIX(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + // 1st miss + rx, err := CompilePOSIX("^abc.*$") + if err != nil { + t.Fatal(err) + } + if rx.FromCache { + t.Error("Regexp shoul not be from cache") + } + if !rx.MatchString("abcxyz") { + t.Error("String didn't match to compiled regexp: ", rx.String()) + } + // 2nd hit + rx2, err := CompilePOSIX("^abc.*$") + if err != nil { + t.Fatal(err) + } + if !rx2.FromCache { + t.Error("Regexp shoul be from cache") + } + if !rx2.MatchString("abcxyz") { + t.Error("String didn't match to compiled regexp: ", rx2.String()) + } +} + +func BenchmarkRegExpCompilePOSIX(b *testing.B) { + ResetCache(defaultCacheItemTTL, true) + + b.ReportAllocs() + + var rx *Regexp + var err error + + for i := 0; i < b.N; i++ { + rx, err = CompilePOSIX("^abc.*$") + if err != nil { + b.Fatal(err) + } + } + + b.Log(rx) +} + +func TestMustCompile(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + // 1st miss + rx := MustCompile("^abc.*$") + if rx.FromCache { + t.Error("Regexp shoul not be from cache") + } + if !rx.MatchString("abcxyz") { + t.Error("String didn't match to compiled regexp: ", rx.String()) + } + // 2nd hit + rx2 := MustCompile("^abc.*$") + if !rx2.FromCache { + t.Error("Regexp shoul be from cache") + } + if !rx2.MatchString("abcxyz") { + t.Error("String didn't match to compiled regexp: ", rx2.String()) + } + // catch panic + panicked := false + func() { + defer func() { + panicked = recover() != nil + }() + MustCompile("*") + }() + if !panicked { + t.Error("Expected panic but it didn't happen") + } +} + +func BenchmarkRegExpMustCompile(b *testing.B) { + ResetCache(defaultCacheItemTTL, true) + + b.ReportAllocs() + + var rx *Regexp + + for i := 0; i < b.N; i++ { + rx = MustCompile("^abc.*$") + } + + b.Log(rx) +} + +func TestMustCompilePOSIX(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + // 1st miss + rx := MustCompilePOSIX("^abc.*$") + if rx.FromCache { + t.Error("Regexp shoul not be from cache") + } + if !rx.MatchString("abcxyz") { + t.Error("String didn't match to compiled regexp: ", rx.String()) + } + // 2nd hit + rx2 := MustCompilePOSIX("^abc.*$") + if !rx2.FromCache { + t.Error("Regexp shoul be from cache") + } + if !rx2.MatchString("abcxyz") { + t.Error("String didn't match to compiled regexp: ", rx2.String()) + } + // catch panic + panicked := false + func() { + defer func() { + panicked = recover() != nil + }() + MustCompilePOSIX("*") + }() + if !panicked { + t.Error("Expected panic but it didn't happen") + } +} + +func BenchmarkRegExpMustCompilePOSIX(b *testing.B) { + ResetCache(defaultCacheItemTTL, true) + b.ReportAllocs() + + var rx *Regexp + + for i := 0; i < b.N; i++ { + rx = MustCompilePOSIX("^abc.*$") + } + + b.Log(rx) +} + +func TestMatchString(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + // 1st miss + matched, err := MatchString("^abc.*$", "abcedfxyz") + if err != nil { + t.Fatal(err) + } + if !matched { + t.Error("String didn't match") + } + // 2nd hit + matched, err = MatchString("^abc.*$", "abcedfxyz") + if err != nil { + t.Fatal(err) + } + if !matched { + t.Error("String didn't match") + } +} + +func TestMatchStringFailed(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + _, err := MatchString("*", "abcedfxyz") + if err == nil { + t.Error("Expected error") + } +} + +func TestMatchStringRegexpNotSet(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + rx := Regexp{} + matched := rx.MatchString("abcdefxyz") + if matched { + t.Error("Expected not to match") + } +} + +func BenchmarkRegExpMatchString(b *testing.B) { + ResetCache(defaultCacheItemTTL, true) + b.ReportAllocs() + + matched := false + var err error + + for i := 0; i < b.N; i++ { + matched, err = MatchString("^abc.*$", "abcdefxyz") + if err != nil { + b.Fatal(err) + } + if !matched { + b.Error("String didn't match") + } + } +} + +func TestMatch(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + data := []byte("abcdefxyz") + // 1st miss + matched, err := Match("^abc.*$", data) + if err != nil { + t.Fatal(err) + } + if !matched { + t.Error("String didn't match") + } + // 2nd hit + matched, err = Match("^abc.*$", data) + if err != nil { + t.Fatal(err) + } + if !matched { + t.Error("String didn't match") + } +} + +func TestMatchFailed(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + _, err := Match("*", []byte("abcdefxyz")) + if err == nil { + t.Error("Expected error") + } +} + +func TestMatchRegexpNotSet(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + rx := Regexp{} + matched := rx.Match([]byte("abcdefxyz")) + if matched { + t.Error("Expected not to match") + } +} + +func BenchmarkRegExpMatch(b *testing.B) { + ResetCache(defaultCacheItemTTL, true) + + b.ReportAllocs() + + data := []byte("abcdefxyz") + + matched := false + var err error + + for i := 0; i < b.N; i++ { + matched, err = Match("^abc.*$", data) + if err != nil { + b.Fatal(err) + } + if !matched { + b.Error("Data didn't match") + } + } +} + +func TestString(t *testing.T) { + rx, err := Compile("^abc.*$") + if err != nil { + t.Fatal(err) + } + if rx.String() != "^abc.*$" { + t.Error("String didn't match to compiled regexp expression") + } +} + +func TestStringRegexpNotSet(t *testing.T) { + rx := Regexp{} + if rx.String() != "" { + t.Error("Empty string expected") + } +} + +func BenchmarkRegExpString(b *testing.B) { + b.ReportAllocs() + + rx, err := Compile("^abc.*$") + if err != nil { + b.Fatal(err) + } + str := "" + + for i := 0; i < b.N; i++ { + str = rx.String() + } + + b.Log(str) +} + +func TestCopy(t *testing.T) { + rx, err := Compile("^abc.*$") + if err != nil { + t.Fatal(err) + } + rxCopy := rx.Copy() + if rx.FromCache != rxCopy.FromCache { + t.Error("Copy's FromCache is not equal") + } + if rx.String() != rxCopy.String() { + t.Error("Copy's Regexp is not equal") + } +} + +func TestReplaceAllString(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + // 1st miss + rx := MustCompile("abc") + newStr := rx.ReplaceAllString("qweabcxyz", "123") + if newStr != "qwe123xyz" { + t.Error("Expected 'qwe123xyz'. Got:", newStr) + } + // 2nd hit + newStr2 := rx.ReplaceAllString("qweabcxyz", "123") + if newStr2 != "qwe123xyz" { + t.Error("Expected 'qwe123xyz'. Got:", newStr2) + } +} + +func TestReplaceAllStringRegexpNotSet(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + rx := &Regexp{} + newStr := rx.ReplaceAllString("qweabcxyz", "123") + if newStr != "" { + t.Error("Expected empty string") + } +} + +func BenchmarkRegexpReplaceAllString(b *testing.B) { + b.ReportAllocs() + + ResetCache(defaultCacheItemTTL, true) + + rx := MustCompile("abc") + str := "" + + for i := 0; i < b.N; i++ { + str = rx.ReplaceAllString("qweabcxyz", "123") + } + + b.Log(str) + +} + +func TestReplaceAllLiteralString(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + // 1st miss + rx := MustCompile("abc") + newStr := rx.ReplaceAllLiteralString("qweabcxyz", "123") + if newStr != "qwe123xyz" { + t.Error("Expected 'qwe123xyz'. Got:", newStr) + } + // 2nd hit + newStr2 := rx.ReplaceAllLiteralString("qweabcxyz", "123") + if newStr2 != "qwe123xyz" { + t.Error("Expected 'qwe123xyz'. Got:", newStr2) + } +} + +func TestReplaceAllLiteralStringRegexpNotSet(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + rx := &Regexp{} + newStr := rx.ReplaceAllLiteralString("qweabcxyz", "123") + if newStr != "" { + t.Error("Expected empty string") + } +} + +func BenchmarkRegexpReplaceAllLiteralString(b *testing.B) { + b.ReportAllocs() + + ResetCache(defaultCacheItemTTL, true) + + rx := MustCompile("abc") + str := "" + + for i := 0; i < b.N; i++ { + str = rx.ReplaceAllLiteralString("qweabcxyz", "123") + } + + b.Log(str) + +} + +func TestReplaceAllStringFunc(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + + f := func(s string) string { + return strings.ToUpper(s) + } + + // 1st miss + rx := MustCompile("abc") + newStr := rx.ReplaceAllStringFunc("qweabcxyz", f) + if newStr != "qweABCxyz" { + t.Error("Expected 'qweABCxyz'. Got:", newStr) + } + // 2nd hit + newStr2 := rx.ReplaceAllStringFunc("qweabcxyz", f) + if newStr2 != "qweABCxyz" { + t.Error("Expected 'qweABCxyz'. Got:", newStr2) + } +} + +func TestReplaceAllStringFuncRegexpNotSet(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + + f := func(s string) string { + return strings.ToUpper(s) + } + + rx := &Regexp{} + newStr := rx.ReplaceAllStringFunc("qweabcxyz", f) + if newStr != "" { + t.Error("Expected empty string returned") + } +} + +func BenchmarkRegexpReplaceAllStringFunc(b *testing.B) { + b.ReportAllocs() + + ResetCache(defaultCacheItemTTL, true) + + f := func(s string) string { + return strings.ToUpper(s) + } + + rx := MustCompile("abc") + str := "" + + for i := 0; i < b.N; i++ { + str = rx.ReplaceAllStringFunc("qweabcxyz", f) + } + + b.Log(str) + +} + +func TestFindAllString(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + + // 1st miss + rx := MustCompile("abc") + res := rx.FindAllString("qweabcxyzabc123abcz", -1) + expectedRes := []string{"abc", "abc", "abc"} + if !reflect.DeepEqual(res, expectedRes) { + t.Error("Expected :", expectedRes, " Got:", res) + } + // 2nd hit + res2 := rx.FindAllString("qweabcxyzabc123abcz", -1) + if !reflect.DeepEqual(res2, expectedRes) { + t.Error("Expected :", expectedRes, " Got:", res2) + } +} + +func TestFindAllStringRegexpNotSet(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + + rx := &Regexp{} + + res := rx.FindAllString("qweabcxyzabc123abcz", -1) + if len(res) > 0 { + t.Error("Expected 0 length slice returned. Got:", res) + } +} + +func BenchmarkRegexpFindAllString(b *testing.B) { + b.ReportAllocs() + + ResetCache(defaultCacheItemTTL, true) + + rx := MustCompile("abc") + var res []string + + for i := 0; i < b.N; i++ { + res = rx.FindAllString("qweabcxyzabc123abcz", -1) + } + + b.Log(res) +} + +func TestFindAllStringSubmatch(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + + // 1st miss + rx := MustCompile("abc") + res := rx.FindAllStringSubmatch("qweabcxyzabc123abcz", -1) + expectedRes := [][]string{ + {"abc"}, + {"abc"}, + {"abc"}, + } + if !reflect.DeepEqual(res, expectedRes) { + t.Error("Expected :", expectedRes, " Got:", res) + } + // 2nd hit + res2 := rx.FindAllStringSubmatch("qweabcxyzabc123abcz", -1) + if !reflect.DeepEqual(res2, expectedRes) { + t.Error("Expected :", expectedRes, " Got:", res2) + } +} + +func TestTestFindAllStringSubmatchRegexpNotSet(t *testing.T) { + ResetCache(defaultCacheItemTTL, true) + + rx := &Regexp{} + + res := rx.FindAllStringSubmatch("qweabcxyzabc123abcz", -1) + if len(res) > 0 { + t.Error("Expected 0 length slice returned. Got:", res) + } +} + +func BenchmarkRegexpFindAllStringSubmatch(b *testing.B) { + b.ReportAllocs() + + ResetCache(defaultCacheItemTTL, true) + + rx := MustCompile("abc") + var res [][]string + + for i := 0; i < b.N; i++ { + res = rx.FindAllStringSubmatch("qweabcxyzabc123abcz", -1) + } + + b.Log(res) +} diff --git a/reverse_proxy.go b/reverse_proxy.go index d710a9ea597..cb880ee48c0 100644 --- a/reverse_proxy.go +++ b/reverse_proxy.go @@ -21,7 +21,6 @@ import ( "net" "net/http" "net/url" - "regexp" "strconv" "strings" "sync" @@ -32,6 +31,7 @@ import ( "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/regexp" "github.com/TykTechnologies/tyk/user" ) From 85f5b90d1588db886e71040037fc19bd35d0e93a Mon Sep 17 00:00:00 2001 From: joshblakeley Date: Wed, 6 Jun 2018 16:16:11 +0100 Subject: [PATCH 42/54] remove line setting tags to nil if none to add from policy --- middleware.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/middleware.go b/middleware.go index 4d88b4625a2..2b324034646 100644 --- a/middleware.go +++ b/middleware.go @@ -276,8 +276,6 @@ func (t BaseMiddleware) ApplyPolicies(key string, session *user.SessionState) er for tag := range tags { session.Tags = append(session.Tags, tag) } - } else { - session.Tags = nil } session.AccessRights = rights From 43e92c4fd7a7d1244fc4cd62673511120bb0c4f7 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Thu, 5 Apr 2018 17:49:19 -0400 Subject: [PATCH 43/54] jsvm: make TykJSHttpResponse fields compatible with BatchReplyUnit (#1519) --- mw_js_plugin.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/mw_js_plugin.go b/mw_js_plugin.go index 48272b5950f..c8570da6e48 100644 --- a/mw_js_plugin.go +++ b/mw_js_plugin.go @@ -395,6 +395,11 @@ type TykJSHttpResponse struct { Code int Body string Headers map[string][]string + + // Make this compatible with BatchReplyUnit + CodeComp int `json:"code"` + BodyComp string `json:"body"` + HeadersComp map[string][]string `json:"headers"` } func (j *JSVM) LoadTykJSApi() { @@ -512,10 +517,14 @@ func (j *JSVM) LoadTykJSApi() { } body, _ := ioutil.ReadAll(resp.Body) + bodyStr := string(body) tykResp := TykJSHttpResponse{ - Code: resp.StatusCode, - Body: string(body), - Headers: resp.Header, + Code: resp.StatusCode, + Body: bodyStr, + Headers: resp.Header, + CodeComp: resp.StatusCode, + BodyComp: bodyStr, + HeadersComp: resp.Header, } retAsStr, _ := json.Marshal(tykResp) From e2e77ec59d34df13075d53f9e9b309aef93e9530 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Thu, 5 Apr 2018 17:50:23 -0400 Subject: [PATCH 44/54] jsvm: add a test for TykMakeHttpRequest --- mw_js_plugin_test.go | 70 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/mw_js_plugin_test.go b/mw_js_plugin_test.go index bd20a8a6900..a93e1f40f25 100644 --- a/mw_js_plugin_test.go +++ b/mw_js_plugin_test.go @@ -321,10 +321,8 @@ func TestJSVMRequestScheme(t *testing.T) { const js = ` var leakMid = new TykJS.TykMiddleware.NewMiddleware({}) - leakMid.NewProcessRequest(function(request, session) { var test = request.Scheme += " appended" - var responseObject = { Body: test, Code: 200 @@ -347,3 +345,71 @@ leakMid.NewProcessRequest(function(request, session) { want, got) } } + +func TestTykMakeHTTPRequest(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + spec := &APISpec{APIDefinition: &apidef.APIDefinition{}} + spec.ConfigData = map[string]interface{}{ + "base_url": ts.URL, + } + + const js = ` + var testTykMakeHTTPRequest = new TykJS.TykMiddleware.NewMiddleware({}) + + testTykMakeHTTPRequest.NewProcessRequest(function(request, session, spec) { + var newRequest = { + "Method": "GET", + "Headers": {"Accept": "application/json"}, + "Domain": spec.config_data.base_url, + "Resource": "/sample" + } + + var resp = TykMakeHttpRequest(JSON.stringify(newRequest)); + var useableResponse = JSON.parse(resp); + + if(useableResponse.Code > 400) { + request.ReturnOverrides.ResponseCode = useableResponse.code + request.ReturnOverrides.ResponseError = "error" + } + + return testTykMakeHTTPRequest.ReturnData(request, {}) + });` + + dynMid := &DynamicMiddleware{ + BaseMiddleware: BaseMiddleware{spec, nil}, + MiddlewareClassName: "testTykMakeHTTPRequest", + Pre: true, + } + jsvm := JSVM{} + jsvm.Init(nil) + if _, err := jsvm.VM.Run(js); err != nil { + t.Fatalf("failed to set up js plugin: %v", err) + } + dynMid.Spec.JSVM = jsvm + + t.Run("Existing endpoint", func(t *testing.T) { + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/sample" + }) + + r := testReq(t, "GET", "/v1/test-data", nil) + _, code := dynMid.ProcessRequest(nil, r, nil) + if want := 200; code != 200 { + t.Fatalf("wanted code to be %d, got %d", want, code) + } + }) + + t.Run("Nonexistent endpoint", func(t *testing.T) { + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/abc" + }) + + r := testReq(t, "GET", "/v1/test-data", nil) + _, code := dynMid.ProcessRequest(nil, r, nil) + if want := 404; code != 404 { + t.Fatalf("wanted code to be %d, got %d", want, code) + } + }) +} From abc192a9aa2fef7de1651d466bd988758da9a56c Mon Sep 17 00:00:00 2001 From: dencoded <33698537+dencoded@users.noreply.github.com> Date: Wed, 6 Jun 2018 23:26:32 -0400 Subject: [PATCH 45/54] Benchmark default version profiling (#1750) added changes for https://github.com/TykTechnologies/tyk/issues/1635 When I first looked at BenchmarkDefaultVersion I've noticed that we spend some significant time in ErrorHandler and SuccessHandler when analytics is enabled. So I started looking into it and that PR includes changes for analytics as well: - we did lots of slice re-allocations for `record.Tags`, I've added helper function which calculates capacity based on spec and configuration - don't spawn go-routine for `analytics.RecordHit`, write to buffered channel instead which is consumed by pool of workers - added new method to redis storage `AppendToSetPipelined(key string, values []string)` to use pipelined operation in analytics pool worker - pool workers maintain buffer of consumed messages, that buffer is sent to redis when it is full or when there were no new messages for some time but buffer holds some unsent messages - the same config value `pool_size` is used to specify number of pool workers - also there is new config value `records_buffer_size` which specifies buffer size for pool workers (size of each pipeline operation). also, buffered channel depends on this value as well and calculated as `pool_size * records_buffer_size`. The default value for `records_buffer_size` is `1000`, also if value from config is lower it will be forced to `1000` - we stop analytics worker poll gracefully when tyk is about to exit Also there some optimizations specifically for `VersionCheck` middle-ware: - parse version `expires` field at API load time, not at request processing - some small clean ups which potentially improve performance and readability of code benchcmp vs master: ``` benchmark old ns/op new ns/op delta BenchmarkDefaultVersion-8 1120529 1023211 -8.69% benchmark old allocs new allocs delta BenchmarkDefaultVersion-8 1487 1226 -17.55% benchmark old bytes new bytes delta BenchmarkDefaultVersion-8 213900 190022 -11.16% ``` --- .gitignore | 1 + analytics.go | 171 +++++++++++++++++++++++++++----------- api_definition.go | 53 +++++++----- apidef/api_definitions.go | 9 +- config/config.go | 1 + gateway_test.go | 8 ++ handler_error.go | 13 +-- handler_success.go | 28 +++++-- ldap_auth_handler.go | 6 +- lint/schema.go | 3 + main.go | 8 +- mw_version_check.go | 11 ++- rpc_storage_handler.go | 8 ++ storage/redis_cluster.go | 35 ++++++-- storage/storage.go | 1 + 15 files changed, 262 insertions(+), 94 deletions(-) diff --git a/.gitignore b/.gitignore index 31351679099..94f7bb3dead 100644 --- a/.gitignore +++ b/.gitignore @@ -47,5 +47,6 @@ petstore.json coprocess_gen_test.go session_state_gen_test.go __pycache__/ +tyk.test tyk_linux_* diff --git a/analytics.go b/analytics.go index 07d72ac91b8..11e9a08a89a 100644 --- a/analytics.go +++ b/analytics.go @@ -3,9 +3,10 @@ package main import ( "fmt" "net" + "sync" + "sync/atomic" "time" - "github.com/jeffail/tunny" "github.com/oschwald/maxminddb-golang" "gopkg.in/vmihailenco/msgpack.v2" @@ -62,10 +63,13 @@ type GeoData struct { const analyticsKeyName = "tyk-system-analytics" +const ( + minRecordsBufferSize = 1000 + recordsBufferFlushInterval = 200 * time.Millisecond + recordsBufferForcedFlushInterval = 1 * time.Second +) + func (a *AnalyticsRecord) GetGeo(ipStr string) { - if !config.Global().AnalyticsConfig.EnableGeoIP { - return - } // Not great, tightly coupled if analytics.GeoIPDB == nil { return @@ -145,18 +149,21 @@ func (a *AnalyticsRecord) SetExpiry(expiresInSeconds int64) { // RedisAnalyticsHandler will record analytics data to a redis back end // as defined in the Config object type RedisAnalyticsHandler struct { - Store storage.Handler - Clean Purger - GeoIPDB *maxminddb.Reader - - AnalyticsPool *tunny.WorkPool + Store storage.Handler + Clean Purger + GeoIPDB *maxminddb.Reader + globalConf config.Config + recordsChan chan *AnalyticsRecord + workerBufferSize uint64 + shouldStop uint32 + poolWg sync.WaitGroup } -func (r *RedisAnalyticsHandler) Init() { - var err error - if config.Global().AnalyticsConfig.EnableGeoIP { - db, err := maxminddb.Open(config.Global().AnalyticsConfig.GeoIPDBLocation) - if err != nil { +func (r *RedisAnalyticsHandler) Init(globalConf config.Config) { + r.globalConf = globalConf + + if r.globalConf.AnalyticsConfig.EnableGeoIP { + if db, err := maxminddb.Open(r.globalConf.AnalyticsConfig.GeoIPDBLocation); err != nil { log.Error("Failed to init GeoIP Database: ", err) } else { r.GeoIPDB = db @@ -165,53 +172,123 @@ func (r *RedisAnalyticsHandler) Init() { analytics.Store.Connect() - ps := config.Global().AnalyticsConfig.PoolSize + ps := r.globalConf.AnalyticsConfig.PoolSize if ps == 0 { ps = 50 } + log.WithField("ps", ps).Debug("Analytics pool workers number") - r.AnalyticsPool, err = tunny.CreatePoolGeneric(ps).Open() - if err != nil { - log.Error("Failed to init analytics pool") + recordsBufferSize := r.globalConf.AnalyticsConfig.RecordsBufferSize + if recordsBufferSize < minRecordsBufferSize { + recordsBufferSize = minRecordsBufferSize // force it to this value } -} + log.WithField("recordsBufferSize", recordsBufferSize).Debug("Analytics total buffer (channel) size") -// RecordHit will store an AnalyticsRecord in Redis -func (r *RedisAnalyticsHandler) RecordHit(record AnalyticsRecord) error { - r.AnalyticsPool.SendWork(func() { - // If we are obfuscating API Keys, store the hashed representation (config check handled in hashing function) - record.APIKey = storage.HashKey(record.APIKey) - - if config.Global().SlaveOptions.UseRPC { - // Extend tag list to include this data so wecan segment by node if necessary - record.Tags = append(record.Tags, "tyk-hybrid-rpc") - } + r.workerBufferSize = recordsBufferSize / uint64(ps) + log.WithField("workerBufferSize", r.workerBufferSize).Debug("Analytics pool worker buffer size") - if config.Global().DBAppConfOptions.NodeIsSegmented { - // Extend tag list to include this data so wecan segment by node if necessary - record.Tags = append(record.Tags, config.Global().DBAppConfOptions.Tags...) - } + r.recordsChan = make(chan *AnalyticsRecord, recordsBufferSize) - // Lets add some metadata - if record.APIKey != "" { - record.Tags = append(record.Tags, "key-"+record.APIKey) - } + // start worker pool + atomic.SwapUint32(&r.shouldStop, 0) + for i := 0; i < ps; i++ { + r.poolWg.Add(1) + go r.recordWorker() + } +} - if record.OrgID != "" { - record.Tags = append(record.Tags, "org-"+record.OrgID) - } +func (r *RedisAnalyticsHandler) Stop() { + // flag to stop sending records into channel + atomic.SwapUint32(&r.shouldStop, 1) - record.Tags = append(record.Tags, "api-"+record.APIID) + // close channel to stop workers + close(r.recordsChan) - encoded, err := msgpack.Marshal(record) + // wait for all workers to be done + r.poolWg.Wait() +} - if err != nil { - log.Error("Error encoding analytics data: ", err) - } +// RecordHit will store an AnalyticsRecord in Redis +func (r *RedisAnalyticsHandler) RecordHit(record *AnalyticsRecord) error { + // check if we should stop sending records 1st + if atomic.LoadUint32(&r.shouldStop) > 0 { + return nil + } - r.Store.AppendToSet(analyticsKeyName, string(encoded)) - }) + // just send record to channel consumed by pool of workers + // leave all data crunching and Redis I/O work for pool workers + r.recordsChan <- record return nil +} + +func (r *RedisAnalyticsHandler) recordWorker() { + defer r.poolWg.Done() + + // this is buffer to send one pipelined command to redis + // use r.recordsBufferSize as cap to reduce slice re-allocations + recordsBuffer := make([]string, 0, r.workerBufferSize) + + // read records from channel and process + lastSentTs := time.Now() + for { + readyToSend := false + select { + + case record, ok := <-r.recordsChan: + // check if channel was closed and it is time to exit from worker + if !ok { + // send what is left in buffer + r.Store.AppendToSetPipelined(analyticsKeyName, recordsBuffer) + return + } + + // we have new record - prepare it and add to buffer + + // If we are obfuscating API Keys, store the hashed representation (config check handled in hashing function) + record.APIKey = storage.HashKey(record.APIKey) + + if r.globalConf.SlaveOptions.UseRPC { + // Extend tag list to include this data so wecan segment by node if necessary + record.Tags = append(record.Tags, "tyk-hybrid-rpc") + } + + if r.globalConf.DBAppConfOptions.NodeIsSegmented { + // Extend tag list to include this data so we can segment by node if necessary + record.Tags = append(record.Tags, r.globalConf.DBAppConfOptions.Tags...) + } + + // Lets add some metadata + if record.APIKey != "" { + record.Tags = append(record.Tags, "key-"+record.APIKey) + } + + if record.OrgID != "" { + record.Tags = append(record.Tags, "org-"+record.OrgID) + } + + record.Tags = append(record.Tags, "api-"+record.APIID) + + if encoded, err := msgpack.Marshal(record); err != nil { + log.WithError(err).Error("Error encoding analytics data") + } else { + recordsBuffer = append(recordsBuffer, string(encoded)) + } + + // identify that buffer is ready to be sent + readyToSend = uint64(len(recordsBuffer)) == r.workerBufferSize + + case <-time.After(recordsBufferFlushInterval): + // nothing was received for that period of time + // anyways send whatever we have, don't hold data too long in buffer + readyToSend = true + } + // send data to Redis and reset buffer + if len(recordsBuffer) > 0 && (readyToSend || time.Since(lastSentTs) >= recordsBufferForcedFlushInterval) { + r.Store.AppendToSetPipelined(analyticsKeyName, recordsBuffer) + recordsBuffer = make([]string, 0, r.workerBufferSize) + lastSentTs = time.Now() + } + } } diff --git a/api_definition.go b/api_definition.go index 0a020b128f2..fb6ea002629 100644 --- a/api_definition.go +++ b/api_definition.go @@ -164,6 +164,21 @@ var ServiceNonce string // keyed to the Api version name, which is determined during routing to speed up lookups func (a APIDefinitionLoader) MakeSpec(def *apidef.APIDefinition) *APISpec { spec := &APISpec{} + + // parse version expiration time stamps + for key, ver := range def.VersionData.Versions { + if ver.Expires == "" || ver.Expires == "-1" { + continue + } + // calculate the time + if t, err := time.Parse("2006-01-02 15:04", ver.Expires); err != nil { + log.WithError(err).WithField("Expires", ver.Expires).Error("Could not parse expiry date for API") + } else { + ver.ExpiresTs = t + def.VersionData.Versions[key] = ver + } + } + spec.APIDefinition = def // We'll push the default HealthChecker: @@ -874,18 +889,16 @@ func (a *APISpec) URLAllowedAndIgnored(r *http.Request, rxPaths []URLSpec, white if v.MethodActions != nil { // We are using an extended path set, check for the method methodMeta, matchMethodOk := v.MethodActions[r.Method] - if !matchMethodOk { continue } - // Matched the method, check what status it is: - if methodMeta.Action == apidef.NoAction { - // NoAction status means we're not treating this request in any special or exceptional way - return a.getURLStatus(v.Status), nil - } + // Matched the method, check what status it is // TODO: Extend here for additional reply options switch methodMeta.Action { + case apidef.NoAction: + // NoAction status means we're not treating this request in any special or exceptional way + return a.getURLStatus(v.Status), nil case apidef.Reply: return StatusRedirectFlowByReply, &methodMeta default: @@ -1043,7 +1056,7 @@ func (a *APISpec) getVersionFromRequest(r *http.Request) string { return r.URL.Query().Get(a.VersionDefinition.Key) case "url": - uPath := strings.Replace(r.URL.Path, a.Proxy.ListenPath, "", 1) + uPath := r.URL.Path[len(a.Proxy.ListenPath):] // First non-empty part of the path is the version ID for _, part := range strings.Split(uPath, "/") { if part != "" { @@ -1067,16 +1080,15 @@ func (a *APISpec) VersionExpired(versionDef *apidef.VersionInfo) (bool, *time.Ti return false, nil } - // otherwise - calculate the time - t, err := time.Parse("2006-01-02 15:04", versionDef.Expires) - if err != nil { - log.Error("Could not parse expiry date for API, dissallow: ", err) + // otherwise use parsed timestamp + if versionDef.ExpiresTs.IsZero() { + log.Error("Could not parse expiry date for API, disallow") return true, nil } // It's in the past, expire // It's in the future, keep going - return time.Since(t) >= 0, &t + return time.Since(versionDef.ExpiresTs) >= 0, &versionDef.ExpiresTs } // RequestValid will check if an incoming request has valid version @@ -1134,17 +1146,17 @@ func (a *APISpec) Version(r *http.Request) (*apidef.VersionInfo, []URLSpec, bool } else { // Extract Version Info // First checking for if default version is set - vname := a.getVersionFromRequest(r) - if vname == "" && a.VersionData.DefaultVersion != "" { - vname = a.VersionData.DefaultVersion + vName := a.getVersionFromRequest(r) + if vName == "" { + if a.VersionData.DefaultVersion == "" { + return &version, nil, false, VersionNotFound + } + vName = a.VersionData.DefaultVersion ctxSetDefaultVersion(r) } - if vname == "" && a.VersionData.DefaultVersion == "" { - return &version, nil, false, VersionNotFound - } // Load Version Data - General var ok bool - if version, ok = a.VersionData.Versions[vname]; !ok { + if version, ok = a.VersionData.Versions[vName]; !ok { return &version, nil, false, VersionDoesNotExist } } @@ -1155,13 +1167,12 @@ func (a *APISpec) Version(r *http.Request) (*apidef.VersionInfo, []URLSpec, bool // Load path data and whitelist data for version rxPaths, rxOk := a.RxPaths[version.Name] - whiteListStatus, wlOk := a.WhiteListEnabled[version.Name] - if !rxOk { log.Error("no RX Paths found for version ", version.Name) return &version, nil, false, VersionDoesNotExist } + whiteListStatus, wlOk := a.WhiteListEnabled[version.Name] if !wlOk { log.Error("No whitelist data found") return &version, nil, false, VersionWhiteListStatusNotFound diff --git a/apidef/api_definitions.go b/apidef/api_definitions.go index faee754576c..ebd3b5c2356 100644 --- a/apidef/api_definitions.go +++ b/apidef/api_definitions.go @@ -7,6 +7,8 @@ import ( "github.com/lonelycode/osin" "gopkg.in/mgo.v2/bson" + "time" + "github.com/TykTechnologies/gojsonschema" "github.com/TykTechnologies/tyk/regexp" ) @@ -207,9 +209,10 @@ type ExtendedPathsSet struct { } type VersionInfo struct { - Name string `bson:"name" json:"name"` - Expires string `bson:"expires" json:"expires"` - Paths struct { + Name string `bson:"name" json:"name"` + Expires string `bson:"expires" json:"expires"` + ExpiresTs time.Time `bson:"-" json:"-"` + Paths struct { Ignored []string `bson:"ignored" json:"ignored"` WhiteList []string `bson:"white_list" json:"white_list"` BlackList []string `bson:"black_list" json:"black_list"` diff --git a/config/config.go b/config/config.go index 0ca116c0ad5..faec0717c86 100644 --- a/config/config.go +++ b/config/config.go @@ -71,6 +71,7 @@ type AnalyticsConfigConfig struct { GeoIPDBLocation string `json:"geo_ip_db_path"` NormaliseUrls NormalisedURLConfig `json:"normalise_urls"` PoolSize int `json:"pool_size"` + RecordsBufferSize uint64 `json:"records_buffer_size"` StorageExpirationTime int `json:"storage_expiration_time"` ignoredIPsCompiled map[string]bool } diff --git a/gateway_test.go b/gateway_test.go index d868ce1e31f..66606c990ae 100644 --- a/gateway_test.go +++ b/gateway_test.go @@ -592,6 +592,8 @@ func TestAnalytics(t *testing.T) { }) // Cleanup before test + // let records to to be sent + time.Sleep(recordsBufferFlushInterval + 50) analytics.Store.GetAndDeleteSet(analyticsKeyName) t.Run("Log errors", func(t *testing.T) { @@ -600,6 +602,9 @@ func TestAnalytics(t *testing.T) { {Path: "/", Code: 401}, }...) + // let records to to be sent + time.Sleep(recordsBufferFlushInterval + 50) + results := analytics.Store.GetAndDeleteSet(analyticsKeyName) if len(results) != 2 { t.Error("Should return 2 record", len(results)) @@ -623,6 +628,9 @@ func TestAnalytics(t *testing.T) { Path: "/", Headers: authHeaders, Code: 200, }) + // let records to to be sent + time.Sleep(recordsBufferFlushInterval + 50) + results := analytics.Store.GetAndDeleteSet(analyticsKeyName) if len(results) != 1 { t.Error("Should return 1 record: ", len(results)) diff --git a/handler_error.go b/handler_error.go index 463983e117b..69ca1ba4b81 100644 --- a/handler_error.go +++ b/handler_error.go @@ -103,13 +103,12 @@ func (e *ErrorHandler) HandleError(w http.ResponseWriter, r *http.Request, errMs } oauthClientID := "" - tags := make([]string, 0) session := ctxGetSession(r) - + tags := make([]string, 0, estimateTagsCapacity(session, e.Spec)) if session != nil { oauthClientID = session.OauthClientID alias = session.Alias - tags = session.Tags + tags = append(tags, session.Tags...) } if len(e.Spec.TagHeaders) > 0 { @@ -159,10 +158,12 @@ func (e *ErrorHandler) HandleError(w http.ResponseWriter, r *http.Request, errMs tags, alias, trackEP, - time.Now(), + t, } - record.GetGeo(ip) + if e.Spec.GlobalConfig.AnalyticsConfig.EnableGeoIP { + record.GetGeo(ip) + } expiresAfter := e.Spec.ExpireAnalyticsAfter if e.Spec.GlobalConfig.EnforceOrgDataAge { @@ -179,7 +180,7 @@ func (e *ErrorHandler) HandleError(w http.ResponseWriter, r *http.Request, errMs record.NormalisePath(&e.Spec.GlobalConfig) } - go analytics.RecordHit(record) + analytics.RecordHit(&record) } // Report in health check diff --git a/handler_success.go b/handler_success.go index 5ddbdd80a7c..ec1eebaa99b 100644 --- a/handler_success.go +++ b/handler_success.go @@ -79,6 +79,21 @@ func addVersionHeader(w http.ResponseWriter, r *http.Request, globalConf config. } } +func estimateTagsCapacity(session *user.SessionState, apiSpec *APISpec) int { + size := 5 // that number of tags expected to be added at least before we record hit + if session != nil { + size += len(session.Tags) + } + + if apiSpec.GlobalConfig.DBAppConfOptions.NodeIsSegmented { + size += len(apiSpec.GlobalConfig.DBAppConfOptions.Tags) + } + + size += len(apiSpec.TagHeaders) + + return size +} + func (s *SuccessHandler) RecordHit(r *http.Request, timing int64, code int, requestCopy *http.Request, responseCopy *http.Response) { if s.Spec.DoNotTrack { @@ -101,13 +116,12 @@ func (s *SuccessHandler) RecordHit(r *http.Request, timing int64, code int, requ // If OAuth, we need to grab it from the session, which may or may not exist oauthClientID := "" - tags := make([]string, 0) var alias string session := ctxGetSession(r) - + tags := make([]string, 0, estimateTagsCapacity(session, s.Spec)) if session != nil { oauthClientID = session.OauthClientID - tags = session.Tags + tags = append(tags, session.Tags...) alias = session.Alias } @@ -171,10 +185,12 @@ func (s *SuccessHandler) RecordHit(r *http.Request, timing int64, code int, requ tags, alias, trackEP, - time.Now(), + t, } - record.GetGeo(ip) + if s.Spec.GlobalConfig.AnalyticsConfig.EnableGeoIP { + record.GetGeo(ip) + } expiresAfter := s.Spec.ExpireAnalyticsAfter if s.Spec.GlobalConfig.EnforceOrgDataAge { @@ -191,7 +207,7 @@ func (s *SuccessHandler) RecordHit(r *http.Request, timing int64, code int, requ record.NormalisePath(&s.Spec.GlobalConfig) } - go analytics.RecordHit(record) + analytics.RecordHit(&record) } // Report in health check diff --git a/ldap_auth_handler.go b/ldap_auth_handler.go index 635d054e7a8..e83c9ca67ee 100644 --- a/ldap_auth_handler.go +++ b/ldap_auth_handler.go @@ -4,7 +4,7 @@ import ( "errors" "strings" - ldap "github.com/mavricknz/ldap" + "github.com/mavricknz/ldap" ) // LDAPStorageHandler implements storage.Handler, this is a read-only implementation to access keys from an LDAP service @@ -177,6 +177,10 @@ func (l LDAPStorageHandler) AppendToSet(keyName, value string) { log.Error("Not implemented") } +func (r *LDAPStorageHandler) AppendToSetPipelined(key string, values []string) { + log.Error("Not implemented") +} + func (l LDAPStorageHandler) RemoveFromSet(keyName, value string) { log.Error("Not implemented") } diff --git a/lint/schema.go b/lint/schema.go index 13df8fd9091..40314bd1d5c 100644 --- a/lint/schema.go +++ b/lint/schema.go @@ -98,6 +98,9 @@ const confSchema = `{ "pool_size": { "type": "integer" }, + "records_buffer_size": { + "type": "integer" + }, "storage_expiration_time": { "type": "integer" }, diff --git a/main.go b/main.go index c16a2e68a24..54a8f14c7fe 100644 --- a/main.go +++ b/main.go @@ -167,7 +167,7 @@ func setupGlobals() { analyticsStore := storage.RedisCluster{KeyPrefix: "analytics-"} analytics.Store = &analyticsStore - analytics.Init() + analytics.Init(globalConf) redisPurgeOnce.Do(func() { store := storage.RedisCluster{KeyPrefix: "analytics-"} @@ -1052,6 +1052,12 @@ func main() { mainLog.Info("Stop signal received.") + // stop analytics workers + if config.Global().EnableAnalytics && analytics.Store == nil { + analytics.Stop() + } + + // write pprof profiles writeProfiles() if config.Global().UseDBAppConfigs { diff --git a/mw_version_check.go b/mw_version_check.go index f444d595547..8a753ff00f3 100644 --- a/mw_version_check.go +++ b/mw_version_check.go @@ -42,10 +42,13 @@ func (v *VersionCheck) ProcessRequest(w http.ResponseWriter, r *http.Request, _ if !requestValid { // Fire a versioning failure event v.FireEvent(EventVersionFailure, EventVersionFailureMeta{ - EventMetaDefault: EventMetaDefault{Message: "Attempted access to disallowed version / path.", OriginatingRequest: EncodeRequestToEvent(r)}, - Path: r.URL.Path, - Origin: request.RealIP(r), - Reason: string(stat), + EventMetaDefault: EventMetaDefault{ + Message: "Attempted access to disallowed version / path.", + OriginatingRequest: EncodeRequestToEvent(r), + }, + Path: r.URL.Path, + Origin: request.RealIP(r), + Reason: string(stat), }) return errors.New(string(stat)), http.StatusForbidden } diff --git a/rpc_storage_handler.go b/rpc_storage_handler.go index 1ec5f6e79b0..bb7c5bf0a50 100644 --- a/rpc_storage_handler.go +++ b/rpc_storage_handler.go @@ -734,6 +734,14 @@ func (r *RPCStorageHandler) AppendToSet(keyName, value string) { } } +func (r *RPCStorageHandler) AppendToSetPipelined(key string, values []string) { + // just falls back to AppendToSet + // TODO: introduce new RPC method for pipelined operation + for _, val := range values { + r.AppendToSet(key, val) + } +} + // SetScrollingWindow is used in the rate limiter to handle rate limits fairly. func (r *RPCStorageHandler) SetRollingWindow(keyName string, per int64, val string, pipeline bool) (int, []interface{}) { start := time.Now() // get current time diff --git a/storage/redis_cluster.go b/storage/redis_cluster.go index dadd3dfed2b..4562432fea4 100644 --- a/storage/redis_cluster.go +++ b/storage/redis_cluster.go @@ -558,13 +558,38 @@ func (r RedisCluster) GetAndDeleteSet(keyName string) []interface{} { } func (r RedisCluster) AppendToSet(keyName, value string) { - log.Debug("Pushing to raw key list: ", keyName) - log.Debug("Appending to fixed key list: ", r.fixKey(keyName)) + fixedKey := r.fixKey(keyName) + log.WithField("keyName", keyName).Debug("Pushing to raw key list") + log.WithField("fixedKey", fixedKey).Debug("Appending to fixed key list") r.ensureConnection() - _, err := r.singleton().Do("RPUSH", r.fixKey(keyName), value) + if _, err := r.singleton().Do("RPUSH", fixedKey, value); err != nil { + log.WithError(err).Error("Error trying to append to set keys") + } +} - if err != nil { - log.Error("Error trying to delete keys: ", err) +func (r RedisCluster) AppendToSetPipelined(key string, values []string) { + if len(values) == 0 { + return + } + + fixedKey := r.fixKey(key) + + // prepare pipeline data + pipeLine := make([]rediscluster.ClusterTransaction, len(values)) + for index, val := range values { + pipeLine[index] = rediscluster.ClusterTransaction{ + Cmd: "RPUSH", + Args: []interface{}{ + fixedKey, + val, + }, + } + } + + // send pipelined command to Redis + r.ensureConnection() + if _, err := r.singleton().DoPipeline(pipeLine); err != nil { + log.WithError(err).Error("Error trying to append to set keys") } } diff --git a/storage/storage.go b/storage/storage.go index 8c8f5cc2a12..e5c6ac566ce 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -37,6 +37,7 @@ type Handler interface { GetSet(string) (map[string]string, error) AddToSet(string, string) AppendToSet(string, string) + AppendToSetPipelined(string, []string) GetAndDeleteSet(string) []interface{} RemoveFromSet(string, string) DeleteScanMatch(string) bool From b04d24441fcf700d3e3d6507ae90e3b5d34c7017 Mon Sep 17 00:00:00 2001 From: Ahmet Soormally Date: Fri, 8 Jun 2018 09:17:54 +0100 Subject: [PATCH 46/54] bump go 1.8 to 1.9 version support --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ce622dd26cb..deebb9355fa 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,12 @@ Tyk is a lightweight, open source API Gateway and Management Platform enables you to control who accesses your API, when they access it and how they access it. Tyk will also record detailed analytics on how your users are interacting with your API and when things go wrong. -Go version 1.8 or later is required to build `master`, the current +Go version 1.9 is required to build `master`, the current development version. Tyk is officially supported on `linux/amd64`, `linux/i386` and `linux/arm64`. +Tests are run against both Go versions 1.9 & 1.10, however at present, only Go 1.9 is officially supported. + ## What is an API Gateway? An API Gateway sits in front of your application(s) and manages the heavy lifting of authorisation, access control and throughput limiting to your services. Ideally, From f1062303deccaf1b425092c17d37f51acd2fdaf8 Mon Sep 17 00:00:00 2001 From: dencoded <33698537+dencoded@users.noreply.github.com> Date: Fri, 8 Jun 2018 00:27:37 -0400 Subject: [PATCH 47/54] added optimizations, added mw tests and benchmarks --- mw_context_vars.go | 39 +++---- mw_context_vars_test.go | 222 +++++++++++++++++++++++++++++++--------- 2 files changed, 189 insertions(+), 72 deletions(-) diff --git a/mw_context_vars.go b/mw_context_vars.go index dfd111ec024..8e5aec4f575 100644 --- a/mw_context_vars.go +++ b/mw_context_vars.go @@ -23,37 +23,24 @@ func (m *MiddlewareContextVars) EnabledForSpec() bool { // ProcessRequest will run any checks on the request on the way through the system, return an error to have the chain fail func (m *MiddlewareContextVars) ProcessRequest(w http.ResponseWriter, r *http.Request, _ interface{}) (error, int) { + r.ParseForm() + + contextDataObject := map[string]interface{}{ + "request_data": r.Form, // Form params (map[string][]string) + "headers": map[string][]string(r.Header), + "headers_Host": r.Host, + "path_parts": strings.Split(r.URL.Path, "/"), // Path parts + "path": r.URL.Path, // path data + "remote_addr": request.RealIP(r), // IP + "request_id": uuid.NewV4().String(), //Correlation ID + } - copiedRequest := copyRequest(r) - contextDataObject := make(map[string]interface{}) - - copiedRequest.ParseForm() - - // Form params (map[string][]string) - contextDataObject["request_data"] = copiedRequest.Form - - contextDataObject["headers"] = map[string][]string(copiedRequest.Header) - - for hname, vals := range copiedRequest.Header { + for hname, vals := range r.Header { n := "headers_" + strings.Replace(hname, "-", "_", -1) contextDataObject[n] = vals[0] } - contextDataObject["headers_Host"] = copiedRequest.Host - - // Path parts - segmentedPathArray := strings.Split(copiedRequest.URL.Path, "/") - contextDataObject["path_parts"] = segmentedPathArray - - // path data - contextDataObject["path"] = copiedRequest.URL.Path - - // IP - contextDataObject["remote_addr"] = request.RealIP(copiedRequest) - - //Correlation ID - contextDataObject["request_id"] = uuid.NewV4().String() - for _, c := range copiedRequest.Cookies() { + for _, c := range r.Cookies() { name := "cookies_" + strings.Replace(c.Name, "-", "_", -1) contextDataObject[name] = c.Value } diff --git a/mw_context_vars_test.go b/mw_context_vars_test.go index 21d51b5c114..dd6871dfecc 100644 --- a/mw_context_vars_test.go +++ b/mw_context_vars_test.go @@ -1,16 +1,19 @@ package main import ( - "errors" + "io" "net/http" "net/http/httptest" + "net/url" + "reflect" + "strings" "testing" "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/test" ) -func testPreparetContextVarsMiddleware() { +func testPrepareContextVarsMiddleware() { buildAndLoadAPI(func(spec *APISpec) { spec.Proxy.ListenPath = "/" spec.EnableContextVars = true @@ -32,7 +35,7 @@ func TestContextVarsMiddleware(t *testing.T) { ts := newTykTestServer() defer ts.Close() - testPreparetContextVarsMiddleware() + testPrepareContextVarsMiddleware() ts.Run(t, []test.TestCase{ {Path: "/test/path", Code: 200, BodyMatch: `"X-Remote-Addr":"127.0.0.1"`}, @@ -48,7 +51,7 @@ func BenchmarkContextVarsMiddleware(b *testing.B) { ts := newTykTestServer() defer ts.Close() - testPreparetContextVarsMiddleware() + testPrepareContextVarsMiddleware() for i := 0; i < b.N; i++ { ts.Run(b, []test.TestCase{ @@ -60,53 +63,180 @@ func BenchmarkContextVarsMiddleware(b *testing.B) { } } -func TestMiddlewareContextVars_ProcessRequest_cookies(t *testing.T) { - - req, _ := http.NewRequest(http.MethodGet, "/", nil) - res := httptest.NewRecorder() - - req.Header.Set("Cookie", "abc=123; def=456") - - err, code := (&MiddlewareContextVars{}).ProcessRequest(res, req, nil) - if err != nil { - t.Fatal(err) - } - - if code != http.StatusOK { - t.Fatal(errors.New("non 200 status code")) - } - - ctx := ctxGetData(req) - - if ctx["cookies_abc"].(string) != "123" { - t.Error("abc should be 123") - } +type testContextVarsData struct { + Method string + URL string + Data string + ExpectedCtxDataObject map[string]interface{} + Header http.Header +} - if ctx["cookies_def"].(string) != "456" { - t.Error("def should be 456") +func testPrepareTestContextVarsMiddleware() map[string]testContextVarsData { + return map[string]testContextVarsData{ + "GET with query string": { + Method: http.MethodGet, + URL: "http://abc.com/aaa/bbb/111/222?x=123&y=test", + Header: http.Header{ + "x-header-a": {"A"}, + "x-header-b": {"B"}, + "x-header-c": {"C"}, + }, + ExpectedCtxDataObject: map[string]interface{}{ + "remote_addr": "192.0.2.1", + "request_data": url.Values{ + "x": {"123"}, + "y": {"test"}, + }, + "headers": map[string][]string{ + "x-header-a": {"A"}, + "x-header-b": {"B"}, + "x-header-c": {"C"}, + }, + "headers_x_header_a": "A", + "headers_x_header_b": "B", + "headers_x_header_c": "C", + "headers_Host": "abc.com", + "path_parts": []string{"", "aaa", "bbb", "111", "222"}, + "path": "/aaa/bbb/111/222", + }, + }, + "POST with query string and encoded form data": { + Method: http.MethodPost, + URL: "http://abc.com/aaa/bbb/111/222?x=123&y=test", + Data: "i=1&j=2&str=abc", + Header: http.Header{ + "Content-Type": {"application/x-www-form-urlencoded"}, + "x-header-a": {"A"}, + "x-header-b": {"B"}, + "x-header-c": {"C"}, + }, + ExpectedCtxDataObject: map[string]interface{}{ + "remote_addr": "192.0.2.1", + "request_data": url.Values{ + "x": {"123"}, + "y": {"test"}, + "i": {"1"}, + "j": {"2"}, + "str": {"abc"}, + }, + "headers": map[string][]string{ + "x-header-a": {"A"}, + "x-header-b": {"B"}, + "x-header-c": {"C"}, + "Content-Type": {"application/x-www-form-urlencoded"}, + }, + "headers_x_header_a": "A", + "headers_x_header_b": "B", + "headers_x_header_c": "C", + "headers_Content_Type": "application/x-www-form-urlencoded", + "headers_Host": "abc.com", + "path_parts": []string{"", "aaa", "bbb", "111", "222"}, + "path": "/aaa/bbb/111/222", + }, + }, + "POST with query string and encoded form data and cookies": { + Method: http.MethodPost, + URL: "http://abc.com/aaa/bbb/111/222?x=123&y=test", + Data: "i=1&j=2&str=abc", + Header: http.Header{ + "Content-Type": {"application/x-www-form-urlencoded"}, + "Cookie": {"c-1=cookie1;c-2=cookie2"}, + "x-header-a": {"A"}, + "x-header-b": {"B"}, + "x-header-c": {"C"}, + }, + ExpectedCtxDataObject: map[string]interface{}{ + "remote_addr": "192.0.2.1", + "request_data": url.Values{ + "x": {"123"}, + "y": {"test"}, + "i": {"1"}, + "j": {"2"}, + "str": {"abc"}, + }, + "headers": map[string][]string{ + "x-header-a": {"A"}, + "x-header-b": {"B"}, + "x-header-c": {"C"}, + "Content-Type": {"application/x-www-form-urlencoded"}, + "Cookie": {"c-1=cookie1;c-2=cookie2"}, + }, + "headers_x_header_a": "A", + "headers_x_header_b": "B", + "headers_x_header_c": "C", + "headers_Content_Type": "application/x-www-form-urlencoded", + "headers_Cookie": "c-1=cookie1;c-2=cookie2", + "headers_Host": "abc.com", + "cookies_c_1": "cookie1", + "cookies_c_2": "cookie2", + "path_parts": []string{"", "aaa", "bbb", "111", "222"}, + "path": "/aaa/bbb/111/222", + }, + }, } +} - if ctx["cookies_ghi"] != nil { - t.Error("ghi should be nil") +func TestContextVarsMiddlewareProcessRequest(t *testing.T) { + mw := &MiddlewareContextVars{} + + tests := testPrepareTestContextVarsMiddleware() + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + var bodyReader io.Reader + if test.Data != "" { + bodyReader = strings.NewReader(test.Data) + } + req := httptest.NewRequest(test.Method, test.URL, bodyReader) + req.Header = test.Header + err, code := mw.ProcessRequest(nil, req, nil) + if err != nil { + t.Error(err) + } + if code != http.StatusOK { + t.Errorf("Wrong response code: %d Eexpected 200.", code) + } + + ctxDataObject := ctxGetData(req) + + // check request_id + if _, ok := ctxDataObject["request_id"].(string); !ok { + t.Error("Missing 'request_id' field") + } + + // delete request_if to do DeepEqual + delete(ctxDataObject, "request_id") + + if !reflect.DeepEqual(ctxDataObject, test.ExpectedCtxDataObject) { + t.Errorf("Expected: %v\n Got: %v\n", test.ExpectedCtxDataObject, ctxDataObject) + } + }) } } -func BenchmarkMiddlewareContextVars_ProcessRequest_cookies(b *testing.B) { - b.ReportAllocs() - - req, _ := http.NewRequest(http.MethodGet, "/", nil) - res := httptest.NewRecorder() - - req.Header.Set("Cookie", "abc=123; def=456") - - for i := 0; i < b.N; i++ { - err, code := (&MiddlewareContextVars{}).ProcessRequest(res, req, nil) - if err != nil { - b.Fatal(err) - } - if code != http.StatusOK { - b.Fatal(errors.New("non 200 status code")) - } +func BenchmarkContextVarsMiddlewareProcessRequest(b *testing.B) { + mw := &MiddlewareContextVars{} + tests := testPrepareTestContextVarsMiddleware() + var err error + var code int + for name, test := range tests { + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var bodyReader io.Reader + if test.Data != "" { + bodyReader = strings.NewReader(test.Data) + } + req := httptest.NewRequest(test.Method, test.URL, bodyReader) + req.Header = test.Header + err, code = mw.ProcessRequest(nil, req, nil) + if err != nil { + b.Error(err) + } + if code != http.StatusOK { + b.Errorf("Wrong response code: %d Eexpected 200.", code) + } + } + }) } - } From 0c6df69dcf77a374cb3631d60f48527ce23d9974 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Thu, 31 May 2018 15:20:05 -0400 Subject: [PATCH 48/54] coprocess: use metadata as map[string]string, enhance conversion --- coprocess.go | 16 +-- .../python/coprocess_session_state_pb2.py | 67 ++++++++-- .../ruby/coprocess_session_state_pb.rb | 2 +- coprocess/coprocess_session_state.pb.go | 125 +++++++++--------- coprocess/proto/coprocess_session_state.proto | 2 +- coprocess/python/tyk/middleware.py | 4 +- coprocess_helpers.go | 29 +++- 7 files changed, 155 insertions(+), 90 deletions(-) diff --git a/coprocess.go b/coprocess.go index 549b5b44852..ba49a7d355c 100644 --- a/coprocess.go +++ b/coprocess.go @@ -136,13 +136,6 @@ func (c *CoProcessor) ObjectFromRequest(r *http.Request) *coprocess.Object { if c.HookType != coprocess.HookType_Pre && c.HookType != coprocess.HookType_CustomKeyCheck { if session := ctxGetSession(r); session != nil { object.Session = ProtoSessionState(session) - // If the session contains metadata, add items to the object's metadata map: - if len(session.MetaData) > 0 { - object.Metadata = make(map[string]string) - for k, v := range session.MetaData { - object.Metadata[k] = v.(string) - } - } } } @@ -278,7 +271,10 @@ func (m *CoProcessMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Requ coProcessor.ObjectPostProcess(returnObject, r) - token := returnObject.Metadata["token"] + var token string + if returnObject.Session != nil { + token = returnObject.Session.Metadata["token"] + } // The CP middleware indicates this is a bad auth: if returnObject.Request.ReturnOverrides.ResponseCode > 400 { @@ -320,10 +316,6 @@ func (m *CoProcessMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Requ returnedSession := TykSessionState(returnObject.Session) - // If the returned object contains metadata, add them to the session: - for k, v := range returnObject.Metadata { - returnedSession.MetaData[k] = string(v) - } if extractor == nil { sessionLifetime := returnedSession.Lifetime(m.Spec.SessionLifetime) // This API is not using the ID extractor, but we've got a session: diff --git a/coprocess/bindings/python/coprocess_session_state_pb2.py b/coprocess/bindings/python/coprocess_session_state_pb2.py index b313d51753c..c342655627b 100644 --- a/coprocess/bindings/python/coprocess_session_state_pb2.py +++ b/coprocess/bindings/python/coprocess_session_state_pb2.py @@ -19,7 +19,7 @@ name='coprocess_session_state.proto', package='coprocess', syntax='proto3', - serialized_pb=_b('\n\x1d\x63oprocess_session_state.proto\x12\tcoprocess\"*\n\nAccessSpec\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0f\n\x07methods\x18\x02 \x03(\t\"s\n\x10\x41\x63\x63\x65ssDefinition\x12\x10\n\x08\x61pi_name\x18\x01 \x01(\t\x12\x0e\n\x06\x61pi_id\x18\x02 \x01(\t\x12\x10\n\x08versions\x18\x03 \x03(\t\x12+\n\x0c\x61llowed_urls\x18\x04 \x03(\x0b\x32\x15.coprocess.AccessSpec\"/\n\rBasicAuthData\x12\x10\n\x08password\x18\x01 \x01(\t\x12\x0c\n\x04hash\x18\x02 \x01(\t\"\x19\n\x07JWTData\x12\x0e\n\x06secret\x18\x01 \x01(\t\"!\n\x07Monitor\x12\x16\n\x0etrigger_limits\x18\x01 \x03(\x01\"\xa5\x07\n\x0cSessionState\x12\x12\n\nlast_check\x18\x01 \x01(\x03\x12\x11\n\tallowance\x18\x02 \x01(\x01\x12\x0c\n\x04rate\x18\x03 \x01(\x01\x12\x0b\n\x03per\x18\x04 \x01(\x01\x12\x0f\n\x07\x65xpires\x18\x05 \x01(\x03\x12\x11\n\tquota_max\x18\x06 \x01(\x03\x12\x14\n\x0cquota_renews\x18\x07 \x01(\x03\x12\x17\n\x0fquota_remaining\x18\x08 \x01(\x03\x12\x1a\n\x12quota_renewal_rate\x18\t \x01(\x03\x12@\n\raccess_rights\x18\n \x03(\x0b\x32).coprocess.SessionState.AccessRightsEntry\x12\x0e\n\x06org_id\x18\x0b \x01(\t\x12\x17\n\x0foauth_client_id\x18\x0c \x01(\t\x12:\n\noauth_keys\x18\r \x03(\x0b\x32&.coprocess.SessionState.OauthKeysEntry\x12\x31\n\x0f\x62\x61sic_auth_data\x18\x0e \x01(\x0b\x32\x18.coprocess.BasicAuthData\x12$\n\x08jwt_data\x18\x0f \x01(\x0b\x32\x12.coprocess.JWTData\x12\x14\n\x0chmac_enabled\x18\x10 \x01(\x08\x12\x13\n\x0bhmac_secret\x18\x11 \x01(\t\x12\x13\n\x0bis_inactive\x18\x12 \x01(\x08\x12\x17\n\x0f\x61pply_policy_id\x18\x13 \x01(\t\x12\x14\n\x0c\x64\x61ta_expires\x18\x14 \x01(\x03\x12#\n\x07monitor\x18\x15 \x01(\x0b\x32\x12.coprocess.Monitor\x12!\n\x19\x65nable_detailed_recording\x18\x16 \x01(\x08\x12\x10\n\x08metadata\x18\x17 \x01(\t\x12\x0c\n\x04tags\x18\x18 \x03(\t\x12\r\n\x05\x61lias\x18\x19 \x01(\t\x12\x14\n\x0clast_updated\x18\x1a \x01(\t\x12\x1d\n\x15id_extractor_deadline\x18\x1b \x01(\x03\x12\x18\n\x10session_lifetime\x18\x1c \x01(\x03\x12\x16\n\x0e\x61pply_policies\x18\x1d \x03(\t\x12\x13\n\x0b\x63\x65rtificate\x18\x1e \x01(\t\x1aP\n\x11\x41\x63\x63\x65ssRightsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.coprocess.AccessDefinition:\x02\x38\x01\x1a\x30\n\x0eOauthKeysEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3') + serialized_pb=_b('\n\x1d\x63oprocess_session_state.proto\x12\tcoprocess\"*\n\nAccessSpec\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0f\n\x07methods\x18\x02 \x03(\t\"s\n\x10\x41\x63\x63\x65ssDefinition\x12\x10\n\x08\x61pi_name\x18\x01 \x01(\t\x12\x0e\n\x06\x61pi_id\x18\x02 \x01(\t\x12\x10\n\x08versions\x18\x03 \x03(\t\x12+\n\x0c\x61llowed_urls\x18\x04 \x03(\x0b\x32\x15.coprocess.AccessSpec\"/\n\rBasicAuthData\x12\x10\n\x08password\x18\x01 \x01(\t\x12\x0c\n\x04hash\x18\x02 \x01(\t\"\x19\n\x07JWTData\x12\x0e\n\x06secret\x18\x01 \x01(\t\"!\n\x07Monitor\x12\x16\n\x0etrigger_limits\x18\x01 \x03(\x01\"\xfd\x07\n\x0cSessionState\x12\x12\n\nlast_check\x18\x01 \x01(\x03\x12\x11\n\tallowance\x18\x02 \x01(\x01\x12\x0c\n\x04rate\x18\x03 \x01(\x01\x12\x0b\n\x03per\x18\x04 \x01(\x01\x12\x0f\n\x07\x65xpires\x18\x05 \x01(\x03\x12\x11\n\tquota_max\x18\x06 \x01(\x03\x12\x14\n\x0cquota_renews\x18\x07 \x01(\x03\x12\x17\n\x0fquota_remaining\x18\x08 \x01(\x03\x12\x1a\n\x12quota_renewal_rate\x18\t \x01(\x03\x12@\n\raccess_rights\x18\n \x03(\x0b\x32).coprocess.SessionState.AccessRightsEntry\x12\x0e\n\x06org_id\x18\x0b \x01(\t\x12\x17\n\x0foauth_client_id\x18\x0c \x01(\t\x12:\n\noauth_keys\x18\r \x03(\x0b\x32&.coprocess.SessionState.OauthKeysEntry\x12\x31\n\x0f\x62\x61sic_auth_data\x18\x0e \x01(\x0b\x32\x18.coprocess.BasicAuthData\x12$\n\x08jwt_data\x18\x0f \x01(\x0b\x32\x12.coprocess.JWTData\x12\x14\n\x0chmac_enabled\x18\x10 \x01(\x08\x12\x13\n\x0bhmac_secret\x18\x11 \x01(\t\x12\x13\n\x0bis_inactive\x18\x12 \x01(\x08\x12\x17\n\x0f\x61pply_policy_id\x18\x13 \x01(\t\x12\x14\n\x0c\x64\x61ta_expires\x18\x14 \x01(\x03\x12#\n\x07monitor\x18\x15 \x01(\x0b\x32\x12.coprocess.Monitor\x12!\n\x19\x65nable_detailed_recording\x18\x16 \x01(\x08\x12\x37\n\x08metadata\x18\x17 \x03(\x0b\x32%.coprocess.SessionState.MetadataEntry\x12\x0c\n\x04tags\x18\x18 \x03(\t\x12\r\n\x05\x61lias\x18\x19 \x01(\t\x12\x14\n\x0clast_updated\x18\x1a \x01(\t\x12\x1d\n\x15id_extractor_deadline\x18\x1b \x01(\x03\x12\x18\n\x10session_lifetime\x18\x1c \x01(\x03\x12\x16\n\x0e\x61pply_policies\x18\x1d \x03(\t\x12\x13\n\x0b\x63\x65rtificate\x18\x1e \x01(\t\x1aP\n\x11\x41\x63\x63\x65ssRightsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.coprocess.AccessDefinition:\x02\x38\x01\x1a\x30\n\x0eOauthKeysEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3') ) @@ -248,8 +248,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1120, - serialized_end=1200, + serialized_start=1159, + serialized_end=1239, ) _SESSIONSTATE_OAUTHKEYSENTRY = _descriptor.Descriptor( @@ -285,8 +285,45 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1202, - serialized_end=1250, + serialized_start=1241, + serialized_end=1289, +) + +_SESSIONSTATE_METADATAENTRY = _descriptor.Descriptor( + name='MetadataEntry', + full_name='coprocess.SessionState.MetadataEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='coprocess.SessionState.MetadataEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='coprocess.SessionState.MetadataEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1291, + serialized_end=1338, ) _SESSIONSTATE = _descriptor.Descriptor( @@ -452,8 +489,8 @@ options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='metadata', full_name='coprocess.SessionState.metadata', index=22, - number=23, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + number=23, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), @@ -509,7 +546,7 @@ ], extensions=[ ], - nested_types=[_SESSIONSTATE_ACCESSRIGHTSENTRY, _SESSIONSTATE_OAUTHKEYSENTRY, ], + nested_types=[_SESSIONSTATE_ACCESSRIGHTSENTRY, _SESSIONSTATE_OAUTHKEYSENTRY, _SESSIONSTATE_METADATAENTRY, ], enum_types=[ ], options=None, @@ -519,18 +556,20 @@ oneofs=[ ], serialized_start=317, - serialized_end=1250, + serialized_end=1338, ) _ACCESSDEFINITION.fields_by_name['allowed_urls'].message_type = _ACCESSSPEC _SESSIONSTATE_ACCESSRIGHTSENTRY.fields_by_name['value'].message_type = _ACCESSDEFINITION _SESSIONSTATE_ACCESSRIGHTSENTRY.containing_type = _SESSIONSTATE _SESSIONSTATE_OAUTHKEYSENTRY.containing_type = _SESSIONSTATE +_SESSIONSTATE_METADATAENTRY.containing_type = _SESSIONSTATE _SESSIONSTATE.fields_by_name['access_rights'].message_type = _SESSIONSTATE_ACCESSRIGHTSENTRY _SESSIONSTATE.fields_by_name['oauth_keys'].message_type = _SESSIONSTATE_OAUTHKEYSENTRY _SESSIONSTATE.fields_by_name['basic_auth_data'].message_type = _BASICAUTHDATA _SESSIONSTATE.fields_by_name['jwt_data'].message_type = _JWTDATA _SESSIONSTATE.fields_by_name['monitor'].message_type = _MONITOR +_SESSIONSTATE.fields_by_name['metadata'].message_type = _SESSIONSTATE_METADATAENTRY DESCRIPTOR.message_types_by_name['AccessSpec'] = _ACCESSSPEC DESCRIPTOR.message_types_by_name['AccessDefinition'] = _ACCESSDEFINITION DESCRIPTOR.message_types_by_name['BasicAuthData'] = _BASICAUTHDATA @@ -589,6 +628,13 @@ # @@protoc_insertion_point(class_scope:coprocess.SessionState.OauthKeysEntry) )) , + + MetadataEntry = _reflection.GeneratedProtocolMessageType('MetadataEntry', (_message.Message,), dict( + DESCRIPTOR = _SESSIONSTATE_METADATAENTRY, + __module__ = 'coprocess_session_state_pb2' + # @@protoc_insertion_point(class_scope:coprocess.SessionState.MetadataEntry) + )) + , DESCRIPTOR = _SESSIONSTATE, __module__ = 'coprocess_session_state_pb2' # @@protoc_insertion_point(class_scope:coprocess.SessionState) @@ -596,10 +642,13 @@ _sym_db.RegisterMessage(SessionState) _sym_db.RegisterMessage(SessionState.AccessRightsEntry) _sym_db.RegisterMessage(SessionState.OauthKeysEntry) +_sym_db.RegisterMessage(SessionState.MetadataEntry) _SESSIONSTATE_ACCESSRIGHTSENTRY.has_options = True _SESSIONSTATE_ACCESSRIGHTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) _SESSIONSTATE_OAUTHKEYSENTRY.has_options = True _SESSIONSTATE_OAUTHKEYSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_SESSIONSTATE_METADATAENTRY.has_options = True +_SESSIONSTATE_METADATAENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) # @@protoc_insertion_point(module_scope) diff --git a/coprocess/bindings/ruby/coprocess_session_state_pb.rb b/coprocess/bindings/ruby/coprocess_session_state_pb.rb index d9a611c4c29..7f638c3e097 100644 --- a/coprocess/bindings/ruby/coprocess_session_state_pb.rb +++ b/coprocess/bindings/ruby/coprocess_session_state_pb.rb @@ -47,7 +47,7 @@ optional :data_expires, :int64, 20 optional :monitor, :message, 21, "coprocess.Monitor" optional :enable_detailed_recording, :bool, 22 - optional :metadata, :string, 23 + map :metadata, :string, :string, 23 repeated :tags, :string, 24 optional :alias, :string, 25 optional :last_updated, :string, 26 diff --git a/coprocess/coprocess_session_state.pb.go b/coprocess/coprocess_session_state.pb.go index c69efce4210..6149fa78cad 100644 --- a/coprocess/coprocess_session_state.pb.go +++ b/coprocess/coprocess_session_state.pb.go @@ -155,7 +155,7 @@ type SessionState struct { DataExpires int64 `protobuf:"varint,20,opt,name=data_expires,json=dataExpires" json:"data_expires,omitempty"` Monitor *Monitor `protobuf:"bytes,21,opt,name=monitor" json:"monitor,omitempty"` EnableDetailedRecording bool `protobuf:"varint,22,opt,name=enable_detailed_recording,json=enableDetailedRecording" json:"enable_detailed_recording,omitempty"` - Metadata string `protobuf:"bytes,23,opt,name=metadata" json:"metadata,omitempty"` + Metadata map[string]string `protobuf:"bytes,23,rep,name=metadata" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` Tags []string `protobuf:"bytes,24,rep,name=tags" json:"tags,omitempty"` Alias string `protobuf:"bytes,25,opt,name=alias" json:"alias,omitempty"` LastUpdated string `protobuf:"bytes,26,opt,name=last_updated,json=lastUpdated" json:"last_updated,omitempty"` @@ -324,11 +324,11 @@ func (m *SessionState) GetEnableDetailedRecording() bool { return false } -func (m *SessionState) GetMetadata() string { +func (m *SessionState) GetMetadata() map[string]string { if m != nil { return m.Metadata } - return "" + return nil } func (m *SessionState) GetTags() []string { @@ -392,63 +392,64 @@ func init() { func init() { proto.RegisterFile("coprocess_session_state.proto", fileDescriptor4) } var fileDescriptor4 = []byte{ - // 914 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x55, 0x6f, 0x4f, 0x1b, 0xc7, - 0x13, 0x96, 0x31, 0x60, 0x7b, 0x6c, 0x03, 0xd9, 0x40, 0xb2, 0x40, 0xf8, 0xfd, 0x8c, 0xa5, 0xa6, - 0x8e, 0x94, 0xa2, 0x96, 0xbe, 0x41, 0x51, 0xa5, 0x36, 0x0d, 0xbc, 0xa0, 0x4d, 0xd2, 0xea, 0x68, - 0xd4, 0x37, 0x95, 0x56, 0xc3, 0xdd, 0x60, 0x6f, 0xb8, 0x7f, 0xdd, 0x5d, 0x63, 0xfc, 0x55, 0xfa, - 0x29, 0xfa, 0x11, 0xab, 0x9d, 0xdb, 0x83, 0x43, 0xb4, 0xef, 0x76, 0x9e, 0xe7, 0xd9, 0xb9, 0x67, - 0x77, 0x67, 0xe6, 0xe0, 0x20, 0x2e, 0x4a, 0x53, 0xc4, 0x64, 0xad, 0xb2, 0x64, 0xad, 0x2e, 0x72, - 0x65, 0x1d, 0x3a, 0x3a, 0x2a, 0x4d, 0xe1, 0x0a, 0xd1, 0xbb, 0xa3, 0xc7, 0x27, 0x00, 0x6f, 0x63, - 0xbf, 0xba, 0x28, 0x29, 0x16, 0x5b, 0xd0, 0x9e, 0x9b, 0x54, 0xb6, 0x46, 0xad, 0x49, 0x2f, 0xf2, - 0x4b, 0x21, 0xa1, 0x93, 0x91, 0x9b, 0x15, 0x89, 0x95, 0x2b, 0xa3, 0xf6, 0xa4, 0x17, 0xd5, 0xe1, - 0xf8, 0xaf, 0x16, 0x6c, 0x55, 0x5b, 0x4f, 0xe9, 0x4a, 0xe7, 0xda, 0xe9, 0x22, 0x17, 0xbb, 0xd0, - 0xc5, 0x52, 0xab, 0x1c, 0x33, 0x0a, 0x59, 0x3a, 0x58, 0xea, 0x8f, 0x98, 0x91, 0xd8, 0x81, 0x75, - 0x4f, 0xe9, 0x44, 0xae, 0x30, 0xb1, 0x86, 0xa5, 0x3e, 0x4f, 0xc4, 0x1e, 0x74, 0x6f, 0xc8, 0x78, - 0x8b, 0x56, 0xb6, 0xf9, 0x0b, 0x77, 0xb1, 0x38, 0x81, 0x01, 0xa6, 0x69, 0xb1, 0xa0, 0x44, 0xcd, - 0x4d, 0x6a, 0xe5, 0xea, 0xa8, 0x3d, 0xe9, 0x1f, 0xef, 0x1c, 0xdd, 0xd9, 0x3f, 0xba, 0xf7, 0x1e, - 0xf5, 0x83, 0xf4, 0x93, 0x49, 0xed, 0xf8, 0x7b, 0x18, 0xfe, 0x88, 0x56, 0xc7, 0x6f, 0xe7, 0x6e, - 0x76, 0x8a, 0x0e, 0xfd, 0x67, 0x4a, 0xb4, 0x76, 0x51, 0x98, 0x24, 0x18, 0xbb, 0x8b, 0x85, 0x80, - 0xd5, 0x19, 0xda, 0x59, 0xf0, 0xc5, 0xeb, 0xf1, 0x21, 0x74, 0x7e, 0xfa, 0xfd, 0x37, 0xde, 0xfa, - 0x0c, 0xd6, 0x2d, 0xc5, 0x86, 0x5c, 0xd8, 0x18, 0xa2, 0xf1, 0xd7, 0xd0, 0xf9, 0x50, 0xe4, 0xda, - 0x15, 0x46, 0x7c, 0x01, 0x1b, 0xce, 0xe8, 0xe9, 0x94, 0x8c, 0x4a, 0x75, 0xa6, 0x9d, 0x95, 0xad, - 0x51, 0x7b, 0xd2, 0x8a, 0x86, 0x01, 0x7d, 0xcf, 0xe0, 0xf8, 0x6f, 0x80, 0xc1, 0x45, 0xf5, 0x1e, - 0x17, 0xfe, 0x39, 0xc4, 0x01, 0x40, 0x8a, 0xd6, 0xa9, 0x78, 0x46, 0xf1, 0x35, 0xa7, 0x6f, 0x47, - 0x3d, 0x8f, 0xbc, 0xf3, 0x80, 0x78, 0x01, 0x3d, 0x3e, 0x14, 0xe6, 0x31, 0xb1, 0xbb, 0x56, 0x74, - 0x0f, 0x78, 0xdb, 0x06, 0x1d, 0xc9, 0x36, 0x13, 0xbc, 0xf6, 0x0f, 0x58, 0x92, 0x91, 0xab, 0x0c, - 0xf9, 0xa5, 0x7f, 0x40, 0xba, 0x2d, 0xb5, 0x21, 0x2b, 0xd7, 0x38, 0x7f, 0x1d, 0x8a, 0x7d, 0xe8, - 0xfd, 0x39, 0x2f, 0x1c, 0xaa, 0x0c, 0x6f, 0xe5, 0x3a, 0x73, 0x5d, 0x06, 0x3e, 0xe0, 0xad, 0x38, - 0x84, 0x41, 0x45, 0x1a, 0xca, 0x69, 0x61, 0x65, 0x87, 0xf9, 0x3e, 0x63, 0x11, 0x43, 0xe2, 0x4b, - 0xd8, 0xac, 0x25, 0x19, 0xea, 0x5c, 0xe7, 0x53, 0xd9, 0x65, 0xd5, 0x46, 0x50, 0x05, 0x54, 0xbc, - 0x06, 0xd1, 0xc8, 0x85, 0xa9, 0x62, 0xdb, 0x3d, 0xd6, 0x6e, 0xdd, 0x67, 0xc4, 0x34, 0xf2, 0x47, - 0xf8, 0x08, 0x43, 0xe4, 0x57, 0x55, 0x46, 0x4f, 0x67, 0xce, 0x4a, 0xe0, 0x57, 0x7f, 0xd5, 0x78, - 0xf5, 0xe6, 0x1d, 0x86, 0x12, 0x88, 0x58, 0x7b, 0x96, 0x3b, 0xb3, 0x8c, 0x06, 0xd8, 0x80, 0x7c, - 0xdd, 0x15, 0x66, 0xea, 0xeb, 0xae, 0x5f, 0xd5, 0x5d, 0x61, 0xa6, 0xe7, 0x89, 0x78, 0x09, 0x9b, - 0x05, 0xce, 0xdd, 0x4c, 0xc5, 0xa9, 0xa6, 0xdc, 0x79, 0x7e, 0xc0, 0xfc, 0x90, 0xe1, 0x77, 0x8c, - 0x9e, 0x27, 0xe2, 0x0c, 0xa0, 0xd2, 0x5d, 0xd3, 0xd2, 0xca, 0x21, 0x7b, 0x79, 0xf9, 0x5f, 0x5e, - 0x7e, 0xf1, 0xca, 0x9f, 0x69, 0x19, 0x8c, 0xf4, 0x8a, 0x3a, 0x16, 0x3f, 0xc0, 0xe6, 0xa5, 0x2f, - 0x48, 0xc5, 0xb9, 0x12, 0x74, 0x28, 0x37, 0x46, 0xad, 0x49, 0xff, 0x58, 0x36, 0x72, 0x3d, 0x28, - 0xd9, 0x68, 0x78, 0xf9, 0xa0, 0x82, 0xbf, 0x82, 0xee, 0xe7, 0x85, 0xab, 0xb6, 0x6e, 0xf2, 0x56, - 0xd1, 0xd8, 0x1a, 0x8a, 0x35, 0xea, 0x7c, 0x5e, 0x38, 0x96, 0x1f, 0xc2, 0x60, 0x96, 0x61, 0xac, - 0x28, 0xc7, 0xcb, 0x94, 0x12, 0xb9, 0x35, 0x6a, 0x4d, 0xba, 0x51, 0xdf, 0x63, 0x67, 0x15, 0x24, - 0xfe, 0x0f, 0x1c, 0xaa, 0x50, 0xdd, 0x4f, 0xf8, 0xf8, 0xe0, 0xa1, 0x0b, 0x46, 0xbc, 0x40, 0x5b, - 0xa5, 0x73, 0x8c, 0x9d, 0xbe, 0x21, 0x29, 0x38, 0x05, 0x68, 0x7b, 0x1e, 0x10, 0x7f, 0x89, 0x58, - 0x96, 0xe9, 0x52, 0x95, 0x45, 0xaa, 0xe3, 0xa5, 0xbf, 0xc4, 0xa7, 0xd5, 0x25, 0x32, 0xfc, 0x2b, - 0xa3, 0xe7, 0x89, 0x37, 0xe3, 0x7d, 0xab, 0xba, 0x12, 0xb7, 0xab, 0x6a, 0xf2, 0xd8, 0x59, 0xa8, - 0xc6, 0xd7, 0xd0, 0xc9, 0xaa, 0x6e, 0x92, 0x3b, 0x8f, 0x4e, 0x17, 0xfa, 0x2c, 0xaa, 0x25, 0xe2, - 0x0d, 0xec, 0x56, 0x07, 0x53, 0x09, 0x39, 0xd4, 0x29, 0x25, 0xca, 0x50, 0x5c, 0x98, 0xc4, 0x57, - 0xe1, 0x33, 0xf6, 0xf9, 0xbc, 0x12, 0x9c, 0x06, 0x3e, 0xaa, 0x69, 0x3f, 0x0a, 0x32, 0x72, 0xc8, - 0x17, 0xf9, 0xbc, 0x1a, 0x05, 0x75, 0xec, 0x7b, 0xca, 0xe1, 0xd4, 0x4a, 0xc9, 0x93, 0x88, 0xd7, - 0x62, 0x1b, 0xd6, 0x30, 0xd5, 0x68, 0xe5, 0x6e, 0x98, 0x5b, 0x3e, 0xf0, 0x47, 0xe2, 0xd6, 0x9d, - 0x97, 0x09, 0x3a, 0x4a, 0xe4, 0x1e, 0x93, 0x7d, 0x8f, 0x7d, 0xaa, 0x20, 0x71, 0x0c, 0x3b, 0x3a, - 0x51, 0x74, 0xeb, 0x0c, 0xc6, 0xae, 0x30, 0x2a, 0x21, 0x4c, 0x52, 0x9d, 0x93, 0xdc, 0xe7, 0xe3, - 0x3f, 0xd5, 0xc9, 0x59, 0xcd, 0x9d, 0x06, 0x4a, 0xbc, 0x82, 0xad, 0x7a, 0x62, 0xa7, 0xfa, 0x8a, - 0x9c, 0xce, 0x48, 0xbe, 0x60, 0xf9, 0x66, 0xc0, 0xdf, 0x07, 0xd8, 0x0f, 0x9d, 0xc6, 0xe5, 0x6b, - 0xb2, 0xf2, 0x80, 0x5d, 0x37, 0xee, 0x5e, 0x93, 0x15, 0x23, 0xe8, 0xc7, 0x64, 0x9c, 0xbe, 0xd2, - 0xb1, 0x6f, 0xbb, 0xff, 0x55, 0x3e, 0x1b, 0xd0, 0xde, 0x1f, 0xf0, 0xe4, 0x51, 0x13, 0xf9, 0x49, - 0x72, 0x4d, 0xcb, 0xfa, 0x57, 0x70, 0x4d, 0x4b, 0xf1, 0x0d, 0xac, 0xdd, 0x60, 0x3a, 0xaf, 0x26, - 0x51, 0xff, 0x78, 0xff, 0xd1, 0x18, 0xbe, 0xff, 0x0f, 0x44, 0x95, 0xf2, 0xcd, 0xca, 0x49, 0x6b, - 0xef, 0x3b, 0xd8, 0x78, 0xd8, 0x16, 0xff, 0x92, 0x7a, 0xbb, 0x99, 0xba, 0xd7, 0xd8, 0x7d, 0xb9, - 0xce, 0x7f, 0xac, 0x6f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x93, 0xd4, 0x16, 0x3a, 0xd2, 0x06, - 0x00, 0x00, + // 933 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0x4f, 0x6f, 0x1b, 0xb7, + 0x13, 0x85, 0x2c, 0xdb, 0x92, 0x66, 0x25, 0x5b, 0x61, 0xec, 0x84, 0xb6, 0xe3, 0xdf, 0x4f, 0x16, + 0x90, 0x54, 0x01, 0x52, 0xa3, 0x75, 0x2f, 0x46, 0x5a, 0xa0, 0x75, 0x63, 0x1d, 0xd4, 0xc6, 0x69, + 0xb1, 0x6e, 0xd0, 0x4b, 0x01, 0x82, 0xde, 0x1d, 0x4b, 0x8c, 0xf7, 0x5f, 0x49, 0xca, 0xb2, 0xbe, + 0x47, 0x4f, 0xfd, 0xb4, 0x05, 0x67, 0xb9, 0xf2, 0x1a, 0x6e, 0x0e, 0xbd, 0x71, 0xde, 0x7b, 0x33, + 0xfb, 0xc8, 0x19, 0x72, 0xe1, 0x30, 0xca, 0x0b, 0x9d, 0x47, 0x68, 0x8c, 0x30, 0x68, 0x8c, 0xca, + 0x33, 0x61, 0xac, 0xb4, 0x78, 0x5c, 0xe8, 0xdc, 0xe6, 0xac, 0xb3, 0xa2, 0x87, 0xa7, 0x00, 0x67, + 0x91, 0x5b, 0x5d, 0x16, 0x18, 0xb1, 0x3e, 0x34, 0xe7, 0x3a, 0xe1, 0x8d, 0x41, 0x63, 0xd4, 0x09, + 0xdd, 0x92, 0x71, 0x68, 0xa5, 0x68, 0x67, 0x79, 0x6c, 0xf8, 0xda, 0xa0, 0x39, 0xea, 0x84, 0x55, + 0x38, 0xfc, 0xbb, 0x01, 0xfd, 0x32, 0xf5, 0x1c, 0xaf, 0x55, 0xa6, 0xac, 0xca, 0x33, 0xb6, 0x07, + 0x6d, 0x59, 0x28, 0x91, 0xc9, 0x14, 0x7d, 0x95, 0x96, 0x2c, 0xd4, 0x07, 0x99, 0x22, 0xdb, 0x85, + 0x4d, 0x47, 0xa9, 0x98, 0xaf, 0x11, 0xb1, 0x21, 0x0b, 0x35, 0x89, 0xd9, 0x3e, 0xb4, 0x6f, 0x51, + 0x3b, 0x8b, 0x86, 0x37, 0xe9, 0x0b, 0xab, 0x98, 0x9d, 0x42, 0x57, 0x26, 0x49, 0xbe, 0xc0, 0x58, + 0xcc, 0x75, 0x62, 0xf8, 0xfa, 0xa0, 0x39, 0x0a, 0x4e, 0x76, 0x8f, 0x57, 0xf6, 0x8f, 0xef, 0xbd, + 0x87, 0x81, 0x97, 0x7e, 0xd4, 0x89, 0x19, 0x7e, 0x0f, 0xbd, 0x1f, 0xa5, 0x51, 0xd1, 0xd9, 0xdc, + 0xce, 0xce, 0xa5, 0x95, 0xee, 0x33, 0x85, 0x34, 0x66, 0x91, 0xeb, 0xd8, 0x1b, 0x5b, 0xc5, 0x8c, + 0xc1, 0xfa, 0x4c, 0x9a, 0x99, 0xf7, 0x45, 0xeb, 0xe1, 0x11, 0xb4, 0x7e, 0xfa, 0xfd, 0x37, 0x4a, + 0x7d, 0x06, 0x9b, 0x06, 0x23, 0x8d, 0xd6, 0x27, 0xfa, 0x68, 0xf8, 0x15, 0xb4, 0x2e, 0xf2, 0x4c, + 0xd9, 0x5c, 0xb3, 0x97, 0xb0, 0x65, 0xb5, 0x9a, 0x4e, 0x51, 0x8b, 0x44, 0xa5, 0xca, 0x1a, 0xde, + 0x18, 0x34, 0x47, 0x8d, 0xb0, 0xe7, 0xd1, 0xf7, 0x04, 0x0e, 0xff, 0x0a, 0xa0, 0x7b, 0x59, 0xf6, + 0xe3, 0xd2, 0xb5, 0x83, 0x1d, 0x02, 0x24, 0xd2, 0x58, 0x11, 0xcd, 0x30, 0xba, 0xa1, 0xf2, 0xcd, + 0xb0, 0xe3, 0x90, 0x77, 0x0e, 0x60, 0x2f, 0xa0, 0x43, 0x9b, 0x92, 0x59, 0x84, 0xe4, 0xae, 0x11, + 0xde, 0x03, 0xce, 0xb6, 0x96, 0x16, 0x79, 0x93, 0x08, 0x5a, 0xbb, 0x06, 0x16, 0xa8, 0xf9, 0x3a, + 0x41, 0x6e, 0xe9, 0x1a, 0x88, 0x77, 0x85, 0xd2, 0x68, 0xf8, 0x06, 0xd5, 0xaf, 0x42, 0x76, 0x00, + 0x9d, 0x3f, 0xe7, 0xb9, 0x95, 0x22, 0x95, 0x77, 0x7c, 0x93, 0xb8, 0x36, 0x01, 0x17, 0xf2, 0x8e, + 0x1d, 0x41, 0xb7, 0x24, 0x35, 0x66, 0xb8, 0x30, 0xbc, 0x45, 0x7c, 0x40, 0x58, 0x48, 0x10, 0xfb, + 0x02, 0xb6, 0x2b, 0x49, 0x2a, 0x55, 0xa6, 0xb2, 0x29, 0x6f, 0x93, 0x6a, 0xcb, 0xab, 0x3c, 0xca, + 0xde, 0x00, 0xab, 0xd5, 0x92, 0x89, 0x20, 0xdb, 0x1d, 0xd2, 0xf6, 0xef, 0x2b, 0xca, 0x24, 0x74, + 0x5b, 0xf8, 0x00, 0x3d, 0x49, 0x5d, 0x15, 0x5a, 0x4d, 0x67, 0xd6, 0x70, 0xa0, 0xae, 0xbf, 0xae, + 0x75, 0xbd, 0x7e, 0x86, 0x7e, 0x04, 0x42, 0xd2, 0x8e, 0x33, 0xab, 0x97, 0x61, 0x57, 0xd6, 0x20, + 0x37, 0x77, 0xb9, 0x9e, 0xba, 0xb9, 0x0b, 0xca, 0xb9, 0xcb, 0xf5, 0x74, 0x12, 0xb3, 0x57, 0xb0, + 0x9d, 0xcb, 0xb9, 0x9d, 0x89, 0x28, 0x51, 0x98, 0x59, 0xc7, 0x77, 0x89, 0xef, 0x11, 0xfc, 0x8e, + 0xd0, 0x49, 0xcc, 0xc6, 0x00, 0xa5, 0xee, 0x06, 0x97, 0x86, 0xf7, 0xc8, 0xcb, 0xab, 0xcf, 0x79, + 0xf9, 0xc5, 0x29, 0x7f, 0xc6, 0xa5, 0x37, 0xd2, 0xc9, 0xab, 0x98, 0xfd, 0x00, 0xdb, 0x57, 0x6e, + 0x20, 0x05, 0xd5, 0x8a, 0xa5, 0x95, 0x7c, 0x6b, 0xd0, 0x18, 0x05, 0x27, 0xbc, 0x56, 0xeb, 0xc1, + 0xc8, 0x86, 0xbd, 0xab, 0x07, 0x13, 0xfc, 0x25, 0xb4, 0x3f, 0x2d, 0x6c, 0x99, 0xba, 0x4d, 0xa9, + 0xac, 0x96, 0xea, 0x87, 0x35, 0x6c, 0x7d, 0x5a, 0x58, 0x92, 0x1f, 0x41, 0x77, 0x96, 0xca, 0x48, + 0x60, 0x26, 0xaf, 0x12, 0x8c, 0x79, 0x7f, 0xd0, 0x18, 0xb5, 0xc3, 0xc0, 0x61, 0xe3, 0x12, 0x62, + 0xff, 0x07, 0x0a, 0x85, 0x9f, 0xee, 0x27, 0xb4, 0x7d, 0x70, 0xd0, 0x25, 0x21, 0x4e, 0xa0, 0x8c, + 0x50, 0x99, 0x8c, 0xac, 0xba, 0x45, 0xce, 0xa8, 0x04, 0x28, 0x33, 0xf1, 0x88, 0x3b, 0x44, 0x59, + 0x14, 0xc9, 0x52, 0x14, 0x79, 0xa2, 0xa2, 0xa5, 0x3b, 0xc4, 0xa7, 0xe5, 0x21, 0x12, 0xfc, 0x2b, + 0xa1, 0x93, 0xd8, 0x99, 0x71, 0xbe, 0x45, 0x35, 0x89, 0x3b, 0xe5, 0x34, 0x39, 0x6c, 0xec, 0xa7, + 0xf1, 0x0d, 0xb4, 0xd2, 0xf2, 0x36, 0xf1, 0xdd, 0x47, 0xbb, 0xf3, 0xf7, 0x2c, 0xac, 0x24, 0xec, + 0x2d, 0xec, 0x95, 0x1b, 0x13, 0x31, 0x5a, 0xa9, 0x12, 0x8c, 0x85, 0xc6, 0x28, 0xd7, 0xb1, 0x9b, + 0xc2, 0x67, 0xe4, 0xf3, 0x79, 0x29, 0x38, 0xf7, 0x7c, 0x58, 0xd1, 0xec, 0x0c, 0xda, 0x29, 0x5a, + 0x49, 0x07, 0xf9, 0x9c, 0xfa, 0xf9, 0xf2, 0x73, 0xfd, 0xbc, 0xf0, 0xba, 0xb2, 0x9d, 0xab, 0x34, + 0x77, 0xf5, 0xac, 0x9c, 0x1a, 0xce, 0xe9, 0xc1, 0xa2, 0x35, 0xdb, 0x81, 0x0d, 0x99, 0x28, 0x69, + 0xf8, 0x9e, 0x7f, 0xde, 0x5c, 0xe0, 0x76, 0x4e, 0x37, 0x7c, 0x5e, 0xc4, 0xd2, 0x62, 0xcc, 0xf7, + 0x89, 0x0c, 0x1c, 0xf6, 0xb1, 0x84, 0xd8, 0x09, 0xec, 0xaa, 0x58, 0xe0, 0x9d, 0xd5, 0x32, 0xb2, + 0xb9, 0x16, 0x31, 0xca, 0x38, 0x51, 0x19, 0xf2, 0x03, 0x3a, 0xa5, 0xa7, 0x2a, 0x1e, 0x57, 0xdc, + 0xb9, 0xa7, 0xd8, 0x6b, 0xe8, 0x57, 0x0f, 0x7b, 0xa2, 0xae, 0xd1, 0xaa, 0x14, 0xf9, 0x0b, 0x92, + 0x6f, 0x7b, 0xfc, 0xbd, 0x87, 0xdd, 0xdb, 0x54, 0xeb, 0x91, 0x42, 0xc3, 0x0f, 0xc9, 0x75, 0xad, + 0x45, 0x0a, 0x0d, 0x1b, 0x40, 0x10, 0xa1, 0xb6, 0xea, 0x5a, 0x45, 0xee, 0x76, 0xfe, 0xaf, 0xf4, + 0x59, 0x83, 0xf6, 0xff, 0x80, 0x27, 0x8f, 0xee, 0x9a, 0x7b, 0x70, 0x6e, 0x70, 0x59, 0xfd, 0x31, + 0x6e, 0x70, 0xc9, 0xbe, 0x86, 0x8d, 0x5b, 0x99, 0xcc, 0xcb, 0x07, 0x2b, 0x38, 0x39, 0x78, 0xf4, + 0x5a, 0xdf, 0xff, 0x2e, 0xc2, 0x52, 0xf9, 0x76, 0xed, 0xb4, 0xb1, 0xff, 0x1d, 0x6c, 0x3d, 0xbc, + 0x3d, 0xff, 0x52, 0x7a, 0xa7, 0x5e, 0xba, 0x53, 0xcf, 0xfe, 0x16, 0x7a, 0x0f, 0x7a, 0xf5, 0x5f, + 0x92, 0xaf, 0x36, 0xe9, 0xaf, 0xf8, 0xcd, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xdc, 0xad, 0x0c, + 0x63, 0x36, 0x07, 0x00, 0x00, } diff --git a/coprocess/proto/coprocess_session_state.proto b/coprocess/proto/coprocess_session_state.proto index 36f080127e9..c045bfd10c6 100644 --- a/coprocess/proto/coprocess_session_state.proto +++ b/coprocess/proto/coprocess_session_state.proto @@ -60,7 +60,7 @@ message SessionState { bool enable_detailed_recording = 22; - string metadata = 23; + map metadata = 23; repeated string tags = 24; string alias = 25; diff --git a/coprocess/python/tyk/middleware.py b/coprocess/python/tyk/middleware.py index 44b84ec39be..5ad0cf5ec16 100644 --- a/coprocess/python/tyk/middleware.py +++ b/coprocess/python/tyk/middleware.py @@ -78,7 +78,9 @@ def process(self, handler, object): handler(object, object.spec) return elif handler.arg_count == 4: - object.request, object.session, object.metadata = handler(object.request, object.session, object.metadata, object.spec) + md = object.session.metadata + object.request, object.session, md = handler(object.request, object.session, md, object.spec) + object.session.metadata.MergeFrom(md) elif handler.arg_count == 3: object.request, object.session = handler(object.request, object.session, object.spec) return object diff --git a/coprocess_helpers.go b/coprocess_helpers.go index 64b8d9a3cd3..4361ec10a0b 100644 --- a/coprocess_helpers.go +++ b/coprocess_helpers.go @@ -5,6 +5,8 @@ package main import ( "encoding/json" + "github.com/Sirupsen/logrus" + "github.com/TykTechnologies/tyk/coprocess" "github.com/TykTechnologies/tyk/user" ) @@ -54,10 +56,9 @@ func TykSessionState(session *coprocess.SessionState) *user.SessionState { } metadata := make(map[string]interface{}) - if session.Metadata != "" { - err := json.Unmarshal([]byte(session.Metadata), &metadata) - if err != nil { - log.Error("Error interpreting metadata: ", err) + if session.Metadata != nil { + for k, v := range session.Metadata { + metadata[k] = v } } @@ -129,6 +130,25 @@ func ProtoSessionState(session *user.SessionState) *coprocess.SessionState { TriggerLimits: session.Monitor.TriggerLimits, } + metadata := make(map[string]string) + if len(session.MetaData) > 0 { + for k, v := range session.MetaData { + switch v.(type) { + case string: + metadata[k] = v.(string) + default: + jsonValue, err := json.Marshal(v) + if err != nil { + log.WithFields(logrus.Fields{ + "prefix": "coprocess", + }).WithError(err).Error("Couldn't encode session metadata") + continue + } + metadata[k] = string(jsonValue) + } + } + } + return &coprocess.SessionState{ LastCheck: session.LastCheck, Allowance: session.Allowance, @@ -152,6 +172,7 @@ func ProtoSessionState(session *user.SessionState) *coprocess.SessionState { ApplyPolicies: session.ApplyPolicies, DataExpires: session.DataExpires, Monitor: monitor, + Metadata: metadata, EnableDetailedRecording: session.EnableDetailedRecording, Tags: session.Tags, Alias: session.Alias, From cdb8664061f0de6c1d9f8b6777eac8be05adc0b9 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Tue, 5 Jun 2018 22:38:21 -0400 Subject: [PATCH 49/54] coprocess: fix compatibility with plugins that still use coprocess.Object.Metadata --- coprocess.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/coprocess.go b/coprocess.go index ba49a7d355c..3d7761e7f5f 100644 --- a/coprocess.go +++ b/coprocess.go @@ -273,6 +273,16 @@ func (m *CoProcessMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Requ var token string if returnObject.Session != nil { + // For compatibility purposes, inject coprocess.Object.Metadata fields: + if returnObject.Metadata != nil { + if returnObject.Session.Metadata == nil { + returnObject.Session.Metadata = make(map[string]string) + } + for k, v := range returnObject.Metadata { + returnObject.Session.Metadata[k] = v + } + } + token = returnObject.Session.Metadata["token"] } From 28a81000332c5e9b17471b511d499b48261f5a9d Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Tue, 12 Jun 2018 20:26:30 -0400 Subject: [PATCH 50/54] coprocess: add additional compatibility code --- coprocess.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/coprocess.go b/coprocess.go index 3d7761e7f5f..513c1675a33 100644 --- a/coprocess.go +++ b/coprocess.go @@ -136,6 +136,8 @@ func (c *CoProcessor) ObjectFromRequest(r *http.Request) *coprocess.Object { if c.HookType != coprocess.HookType_Pre && c.HookType != coprocess.HookType_CustomKeyCheck { if session := ctxGetSession(r); session != nil { object.Session = ProtoSessionState(session) + // For compatibility purposes: + object.Metadata = object.Session.Metadata } } From cf3c47b2edf0b56a1e25cb9e6e98c5f1be554d24 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Tue, 12 Jun 2018 20:32:57 -0400 Subject: [PATCH 51/54] coprocess: enhance tests with session metadata checks --- coprocess_grpc_test.go | 128 ++++++++++++++++++++++++++++++++++----- coprocess_python_test.go | 73 ++++++++++++++++++++++ 2 files changed, 187 insertions(+), 14 deletions(-) diff --git a/coprocess_grpc_test.go b/coprocess_grpc_test.go index 135d4abb493..717568f803b 100644 --- a/coprocess_grpc_test.go +++ b/coprocess_grpc_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "net" "net/http" + "strings" "testing" "context" @@ -18,6 +19,7 @@ import ( "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/coprocess" "github.com/TykTechnologies/tyk/test" + "github.com/TykTechnologies/tyk/user" ) const ( @@ -36,6 +38,56 @@ func (d *dispatcher) Dispatch(ctx context.Context, object *coprocess.Object) (*c object.Request.SetHeaders = map[string]string{ testHeaderName: testHeaderValue, } + case "testPostHook1": + testKeyValue, ok := object.Session.Metadata["testkey"] + if !ok { + object.Request.ReturnOverrides.ResponseError = "'testkey' not found in session metadata" + object.Request.ReturnOverrides.ResponseCode = 400 + break + } + jsonObject := make(map[string]string) + if err := json.Unmarshal([]byte(testKeyValue), &jsonObject); err != nil { + object.Request.ReturnOverrides.ResponseError = "couldn't decode 'testkey' nested value" + object.Request.ReturnOverrides.ResponseCode = 400 + break + } + nestedKeyValue, ok := jsonObject["nestedkey"] + if !ok { + object.Request.ReturnOverrides.ResponseError = "'nestedkey' not found in JSON object" + object.Request.ReturnOverrides.ResponseCode = 400 + break + } + if nestedKeyValue != "nestedvalue" { + object.Request.ReturnOverrides.ResponseError = "'nestedvalue' value doesn't match" + object.Request.ReturnOverrides.ResponseCode = 400 + break + } + testKey2Value, ok := object.Session.Metadata["testkey2"] + if !ok { + object.Request.ReturnOverrides.ResponseError = "'testkey' not found in session metadata" + object.Request.ReturnOverrides.ResponseCode = 400 + break + } + if testKey2Value != "testvalue" { + object.Request.ReturnOverrides.ResponseError = "'testkey2' value doesn't match" + object.Request.ReturnOverrides.ResponseCode = 400 + break + } + + // Check for compatibility (object.Metadata should contain the same keys as object.Session.Metadata) + for k, v := range object.Metadata { + sessionKeyValue, ok := object.Session.Metadata[k] + if !ok { + object.Request.ReturnOverrides.ResponseError = k + " not found in object.Session.Metadata" + object.Request.ReturnOverrides.ResponseCode = 400 + break + } + if strings.Compare(sessionKeyValue, v) != 0 { + object.Request.ReturnOverrides.ResponseError = k + " doesn't match value in object.Session.Metadata" + object.Request.ReturnOverrides.ResponseCode = 400 + break + } + } } return object, nil } @@ -50,13 +102,14 @@ func newTestGRPCServer() (s *grpc.Server) { return s } -func loadTestGRPCSpec() *APISpec { - spec := buildAndLoadAPI(func(spec *APISpec) { - spec.APIID = "999999" +func loadTestGRPCAPIs() { + buildAndLoadAPI(func(spec *APISpec) { + spec.APIID = "1" spec.OrgID = "default" spec.Auth = apidef.Auth{ AuthHeaderName: "authorization", } + spec.UseKeylessAccess = false spec.VersionData = struct { NotVersioned bool `bson:"not_versioned" json:"not_versioned"` DefaultVersion string `bson:"default_version" json:"default_version"` @@ -77,9 +130,34 @@ func loadTestGRPCSpec() *APISpec { }, Driver: apidef.GrpcDriver, } - })[0] - - return spec + }, func(spec *APISpec) { + spec.APIID = "2" + spec.OrgID = "default" + spec.Auth = apidef.Auth{ + AuthHeaderName: "authorization", + } + spec.UseKeylessAccess = false + spec.VersionData = struct { + NotVersioned bool `bson:"not_versioned" json:"not_versioned"` + DefaultVersion string `bson:"default_version" json:"default_version"` + Versions map[string]apidef.VersionInfo `bson:"versions" json:"versions"` + }{ + NotVersioned: true, + Versions: map[string]apidef.VersionInfo{ + "v1": { + Name: "v1", + }, + }, + } + spec.Proxy.ListenPath = "/grpc-test-api-2/" + spec.Proxy.StripListenPath = true + spec.CustomMiddleware = apidef.MiddlewareSection{ + Post: []apidef.MiddlewareDefinition{ + {Name: "testPostHook1"}, + }, + Driver: apidef.GrpcDriver, + } + }) } func startTykWithGRPC() (*tykTestServer, *grpc.Server) { @@ -95,8 +173,8 @@ func startTykWithGRPC() (*tykTestServer, *grpc.Server) { } ts := newTykTestServer(tykTestServerConfig{coprocessConfig: cfg}) - // Load a test API: - loadTestGRPCSpec() + // Load test APIs: + loadTestGRPCAPIs() return &ts, grpcServer } @@ -105,11 +183,20 @@ func TestGRPCDispatch(t *testing.T) { defer ts.Close() defer grpcServer.Stop() + keyID := createSession(func(s *user.SessionState) { + s.MetaData = map[string]interface{}{ + "testkey": map[string]interface{}{"nestedkey": "nestedvalue"}, + "testkey2": "testvalue", + } + }) + headers := map[string]string{"authorization": keyID} + t.Run("Pre Hook with SetHeaders", func(t *testing.T) { res, err := ts.Run(t, test.TestCase{ - Path: "/grpc-test-api/", - Method: http.MethodGet, - Code: http.StatusOK, + Path: "/grpc-test-api/", + Method: http.MethodGet, + Code: http.StatusOK, + Headers: headers, }) if err != nil { t.Fatalf("Request failed: %s", err.Error()) @@ -132,6 +219,15 @@ func TestGRPCDispatch(t *testing.T) { } }) + t.Run("Post Hook with metadata", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Path: "/grpc-test-api-2/", + Method: http.MethodGet, + Code: http.StatusOK, + Headers: headers, + }) + }) + } func BenchmarkGRPCDispatch(b *testing.B) { @@ -139,14 +235,18 @@ func BenchmarkGRPCDispatch(b *testing.B) { defer ts.Close() defer grpcServer.Stop() + keyID := createSession(func(s *user.SessionState) {}) + headers := map[string]string{"authorization": keyID} + b.Run("Pre Hook with SetHeaders", func(b *testing.B) { path := "/grpc-test-api/" b.ReportAllocs() for i := 0; i < b.N; i++ { ts.Run(b, test.TestCase{ - Path: path, - Method: http.MethodGet, - Code: http.StatusOK, + Path: path, + Method: http.MethodGet, + Code: http.StatusOK, + Headers: headers, }) } }) diff --git a/coprocess_python_test.go b/coprocess_python_test.go index 86bbcd5674c..77eaaf3ca75 100644 --- a/coprocess_python_test.go +++ b/coprocess_python_test.go @@ -9,6 +9,7 @@ import ( "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/test" + "github.com/TykTechnologies/tyk/user" ) var pythonBundleWithAuthCheck = map[string]string{ @@ -42,6 +43,51 @@ def MyAuthHook(request, session, metadata, spec): `, } +var pythonBundleWithPostHook = map[string]string{ + "manifest.json": ` + { + "file_list": [ + "middleware.py" + ], + "custom_middleware": { + "driver": "python", + "post": [{ + "name": "MyPostHook" + }] + } + } + `, + "middleware.py": ` +from tyk.decorators import * +from gateway import TykGateway as tyk +import json + +@Hook +def MyPostHook(request, session, spec): + print("called", session.metadata) + if "testkey" not in session.metadata.keys(): + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "'testkey' not found in metadata" + return request, session + nested_data = json.loads(session.metadata["testkey"]) + if "nestedkey" not in nested_data: + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "'nestedkey' not found in nested metadata" + return request, session + if "stringkey" not in session.metadata.keys(): + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "'stringkey' not found in metadata" + return request, session + stringkey = session.metadata["stringkey"] + if stringkey != "testvalue": + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "'stringkey' value doesn't match" + return request, session + return request, session + +`, +} + func TestPythonBundles(t *testing.T) { ts := newTykTestServer(tykTestServerConfig{ coprocessConfig: config.CoProcessConfig{ @@ -50,6 +96,7 @@ func TestPythonBundles(t *testing.T) { defer ts.Close() bundleID := registerBundle("python_with_auth_check", pythonBundleWithAuthCheck) + bundleID2 := registerBundle("python_with_post_hook", pythonBundleWithPostHook) t.Run("Single-file bundle with authentication hook", func(t *testing.T) { buildAndLoadAPI(func(spec *APISpec) { @@ -70,4 +117,30 @@ func TestPythonBundles(t *testing.T) { {Path: "/test-api/", Code: 403, Headers: invalidAuth}, }...) }) + + t.Run("Single-file bundle with post hook", func(t *testing.T) { + + keyID := createSession(func(s *user.SessionState) { + s.MetaData = map[string]interface{}{ + "testkey": map[string]interface{}{"nestedkey": "nestedvalue"}, + "stringkey": "testvalue", + } + }) + + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/test-api-2/" + spec.UseKeylessAccess = false + spec.EnableCoProcessAuth = false + spec.CustomMiddlewareBundle = bundleID2 + spec.VersionData.NotVersioned = true + }) + + time.Sleep(1 * time.Second) + + auth := map[string]string{"Authorization": keyID} + + ts.Run(t, []test.TestCase{ + {Path: "/test-api-2/", Code: 200, Headers: auth}, + }...) + }) } From 5bbab7f67be142039b7f67f3289ff17487f76156 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 13 Jun 2018 18:05:27 +0500 Subject: [PATCH 52/54] Do less session updates, remove hash check (#1770) Previously, to know if the session needs to be updated or not, we had to generate some session hash (by serializing an object to msgpack), and if a hash of new session does not match a hash of original session, we triggered an update. This was causing unnecessary updates because change in any field changed hash, and hashing itself was quite slow. There are cases when fields changed in existing session object do not need to trigger updates. And this PR introduce the new way of manually specifying places when storage update is needed for the session or not. If you want update session, just call `ctxScheduleSessionUpdate(r)`, where `r` is current request object. It will set the request context boolean variable, and will schedule update, at the end of the request. As a big bonus, it will restrict running more than one session update per request. The main showcases for this feature are applying policies and quota updates. In case of `ApplyPolicy`, we run this function every time, on each request, so session object always gets fresh data from policy, it does not really depend on data from original session object, except policy_id. Previously when policy got changed, it updated session, its `hash` changed as well, and it was triggering an update for each token. Now, `ApplyPolicies` never triggers session update. And keys when we need to see current key permissions with Tyk API, covered by running `ApplyPolicies` one more time inside API call. In case of quotas, on each request, we change `session.QuotaRemaining` field on each request, which triggers session update too. But in fact, `QuotaRemaining` always computed dynamically, and value from original session used only for Keys API, when you retrieve key and need to know current quota usage. Now, quota updates never trigger session update as well, which means minus one redis call in all cases when rate limiting used. Keys API now separately fetch quota key value, and dynamically update session, before responding to the user. So what actually triggers session update now? 1) Key API methods, when token is created or modified 2) When keys are created via auth plugins 3) When the key is created by JWT/OIDC/oAuth 3) When plugin modify session meta 4) When key received from RPC layer and needs be saved to redis 5) When new Quota period is started, e.g `session.QuotaRenews` updated This PR introduces changes in almost all layers, and basic concepts, and we need to thoroughly test auth plugins, JWT, OIDC, oAuth, Keys API, RPC and etc. ``` benchmark old ns/op new ns/op delta BenchmarkURLReplacer-4 2787 2351 -15.64% BenchmarkTagHeaders-4 1141 1220 +6.92% BenchmarkDefaultVersion-4 1685719 1501562 -10.92% BenchmarkGetVersionFromRequest/Header_location-4 738379 703549 -4.72% BenchmarkGetVersionFromRequest/URL_param_location-4 714396 734280 +2.78% BenchmarkGetVersionFromRequest/URL_location-4 719776 747636 +3.87% BenchmarkApiReload-4 5742972 5858318 +2.01% BenchmarkValueExtractor/HeaderSource-4 1117310 1011728 -9.45% BenchmarkValueExtractor/FormSource-4 1144399 1022131 -10.68% BenchmarkRegexExtractor/HeaderSource-4 1156009 936368 -19.00% BenchmarkRegexExtractor/BodySource-4 1070175 986241 -7.84% BenchmarkRegexExtractor/FormSource-4 1109912 1029310 -7.26% BenchmarkXPathExtractor/HeaderSource-4 1030084 1044817 +1.43% BenchmarkXPathExtractor/BodySource-4 1102783 1014181 -8.03% BenchmarkXPathExtractor/FormSource-4 1119198 1056668 -5.59% BenchmarkInitGenericEventHandlers-4 962 1022 +6.24% BenchmarkMultiSession_BA_Standard_OK-4 2496975 1374786 -44.94% BenchmarkBearerTokenAuthKeySession-4 2692664 1382167 -48.67% BenchmarkMultiAuthBackwardsCompatibleSession-4 2531795 1946584 -23.11% BenchmarkBasicAuth-4 184138687 184018566 -0.07% BenchmarkContextVarsMiddleware-4 1971604 1930676 -2.08% BenchmarkHMACAuthSessionPass-4 1600513 1447617 -9.55% BenchmarkHMACAuthSessionPassWithHeaderField-4 1624976 1492679 -8.14% BenchmarkIPBlacklistMiddleware-4 37213 37694 +1.29% BenchmarkIPMiddlewarePass-4 24961 24570 -1.57% BenchmarkJWTSessionHMAC-4 545083 495488 -9.10% BenchmarkJWTSessionRSA-4 698220 657780 -5.79% BenchmarkJWTSessionRSABearer-4 696516 667341 -4.19% BenchmarkJWTSessionRSAWithRawSourceOnWithClientID-4 703284 679524 -3.38% BenchmarkJWTSessionRSAWithRawSource-4 688620 666049 -3.28% BenchmarkJWTSessionRSAWithJWK-4 706496 665930 -5.74% BenchmarkJWTSessionRSAWithEncodedJWK-4 720011 662621 -7.97% BenchmarkProcessRequestLiveQuotaLimit-4 2553445 2521197 -1.26% BenchmarkProcessRequestOffThreadQuotaLimit-4 2051324 1715698 -16.36% BenchmarkProcessRequestLiveRedisRollingLimiter-4 2232012 2871772 +28.66% BenchmarkProcessRequestOffThreadRedisRollingLimiter-4 1679671 2037269 +21.29% BenchmarkStripAuth_stripFromHeaders-4 10501 10430 -0.68% BenchmarkStripAuth_stripFromParams-4 48499 49301 +1.65% BenchmarkTransformNonAscii-4 20156 20855 +3.47% BenchmarkTransformJSONMarshal-4 21362 21190 -0.81% BenchmarkRewriter-4 46417 47460 +2.25% BenchmarkValidateJSONSchema-4 5449007 5569919 +2.22% BenchmarkVersioning-4 7043440 7119762 +1.08% BenchmarkVirtualEndpoint-4 1531284 1542226 +0.71% BenchmarkApplyPolicies-4 32070 24858 -22.49% BenchmarkResponseHeaderInjection-4 5907229 6085702 +3.02% BenchmarkRequestIPHops-4 570 575 +0.88% BenchmarkWrappedServeHTTP-4 199790 198973 -0.41% BenchmarkCopyRequestResponse-4 22979 23169 +0.83% benchmark old allocs new allocs delta BenchmarkURLReplacer-4 12 12 +0.00% BenchmarkTagHeaders-4 15 15 +0.00% BenchmarkDefaultVersion-4 1407 1340 -4.76% BenchmarkGetVersionFromRequest/Header_location-4 564 564 +0.00% BenchmarkGetVersionFromRequest/URL_param_location-4 548 548 +0.00% BenchmarkGetVersionFromRequest/URL_location-4 529 529 +0.00% BenchmarkApiReload-4 23909 23909 +0.00% BenchmarkValueExtractor/HeaderSource-4 75 74 -1.33% BenchmarkValueExtractor/FormSource-4 131 101 -22.90% BenchmarkRegexExtractor/HeaderSource-4 77 76 -1.30% BenchmarkRegexExtractor/BodySource-4 87 86 -1.15% BenchmarkRegexExtractor/FormSource-4 133 103 -22.56% BenchmarkXPathExtractor/HeaderSource-4 133 132 -0.75% BenchmarkXPathExtractor/BodySource-4 170 140 -17.65% BenchmarkXPathExtractor/FormSource-4 195 164 -15.90% BenchmarkInitGenericEventHandlers-4 8 8 +0.00% BenchmarkMultiSession_BA_Standard_OK-4 528 321 -39.20% BenchmarkBearerTokenAuthKeySession-4 483 289 -40.17% BenchmarkMultiAuthBackwardsCompatibleSession-4 486 266 -45.27% BenchmarkBasicAuth-4 19955 19893 -0.31% BenchmarkContextVarsMiddleware-4 1690 1710 +1.18% BenchmarkHMACAuthSessionPass-4 294 268 -8.84% BenchmarkHMACAuthSessionPassWithHeaderField-4 322 299 -7.14% BenchmarkIPBlacklistMiddleware-4 152 152 +0.00% BenchmarkIPMiddlewarePass-4 95 95 +0.00% BenchmarkJWTSessionHMAC-4 537 499 -7.08% BenchmarkJWTSessionRSA-4 645 608 -5.74% BenchmarkJWTSessionRSABearer-4 647 610 -5.72% BenchmarkJWTSessionRSAWithRawSourceOnWithClientID-4 667 629 -5.70% BenchmarkJWTSessionRSAWithRawSource-4 657 627 -4.57% BenchmarkJWTSessionRSAWithJWK-4 660 630 -4.55% BenchmarkJWTSessionRSAWithEncodedJWK-4 665 635 -4.51% BenchmarkProcessRequestLiveQuotaLimit-4 776 746 -3.87% BenchmarkProcessRequestOffThreadQuotaLimit-4 593 564 -4.89% BenchmarkProcessRequestLiveRedisRollingLimiter-4 1268 1233 -2.76% BenchmarkProcessRequestOffThreadRedisRollingLimiter-4 1437 1329 -7.52% BenchmarkStripAuth_stripFromHeaders-4 66 66 +0.00% BenchmarkStripAuth_stripFromParams-4 220 220 +0.00% BenchmarkTransformNonAscii-4 98 98 +0.00% BenchmarkTransformJSONMarshal-4 101 101 +0.00% BenchmarkRewriter-4 238 238 +0.00% BenchmarkValidateJSONSchema-4 2986 2995 +0.30% BenchmarkVersioning-4 2823 2784 -1.38% BenchmarkVirtualEndpoint-4 467 468 +0.21% BenchmarkApplyPolicies-4 102 86 -15.69% BenchmarkResponseHeaderInjection-4 2804 2813 +0.32% BenchmarkRequestIPHops-4 4 4 +0.00% BenchmarkWrappedServeHTTP-4 144 144 +0.00% BenchmarkCopyRequestResponse-4 76 76 +0.00% benchmark old bytes new bytes delta BenchmarkURLReplacer-4 1088 1088 +0.00% BenchmarkTagHeaders-4 384 384 +0.00% BenchmarkDefaultVersion-4 198254 190056 -4.14% BenchmarkGetVersionFromRequest/Header_location-4 86482 86351 -0.15% BenchmarkGetVersionFromRequest/URL_param_location-4 86058 86034 -0.03% BenchmarkGetVersionFromRequest/URL_location-4 83326 83288 -0.05% BenchmarkApiReload-4 1789294 1789305 +0.00% BenchmarkValueExtractor/HeaderSource-4 5702 5184 -9.08% BenchmarkValueExtractor/FormSource-4 13095 11279 -13.87% BenchmarkRegexExtractor/HeaderSource-4 5744 5208 -9.33% BenchmarkRegexExtractor/BodySource-4 10188 9667 -5.11% BenchmarkRegexExtractor/FormSource-4 13133 11319 -13.81% BenchmarkXPathExtractor/HeaderSource-4 10498 9992 -4.82% BenchmarkXPathExtractor/BodySource-4 16220 14404 -11.20% BenchmarkXPathExtractor/FormSource-4 18427 16599 -9.92% BenchmarkInitGenericEventHandlers-4 528 528 +0.00% BenchmarkMultiSession_BA_Standard_OK-4 81775 60464 -26.06% BenchmarkBearerTokenAuthKeySession-4 77359 58083 -24.92% BenchmarkMultiAuthBackwardsCompatibleSession-4 78924 58578 -25.78% BenchmarkBasicAuth-4 1043489 1036911 -0.63% BenchmarkContextVarsMiddleware-4 279017 280941 +0.69% BenchmarkHMACAuthSessionPass-4 60591 56631 -6.54% BenchmarkHMACAuthSessionPassWithHeaderField-4 62726 60034 -4.29% BenchmarkIPBlacklistMiddleware-4 32311 32313 +0.01% BenchmarkIPMiddlewarePass-4 26716 26712 -0.01% BenchmarkJWTSessionHMAC-4 81524 75894 -6.91% BenchmarkJWTSessionRSA-4 103848 96851 -6.74% BenchmarkJWTSessionRSABearer-4 105040 98076 -6.63% BenchmarkJWTSessionRSAWithRawSourceOnWithClientID-4 105879 99197 -6.31% BenchmarkJWTSessionRSAWithRawSource-4 103294 99034 -4.12% BenchmarkJWTSessionRSAWithJWK-4 102284 98035 -4.15% BenchmarkJWTSessionRSAWithEncodedJWK-4 102509 98261 -4.14% BenchmarkProcessRequestLiveQuotaLimit-4 92803 88794 -4.32% BenchmarkProcessRequestOffThreadQuotaLimit-4 84241 80573 -4.35% BenchmarkProcessRequestLiveRedisRollingLimiter-4 108492 108089 -0.37% BenchmarkProcessRequestOffThreadRedisRollingLimiter-4 113996 111109 -2.53% BenchmarkStripAuth_stripFromHeaders-4 2801 2801 +0.00% BenchmarkStripAuth_stripFromParams-4 14164 14164 +0.00% BenchmarkTransformNonAscii-4 11059 11059 +0.00% BenchmarkTransformJSONMarshal-4 11508 11507 -0.01% BenchmarkRewriter-4 45598 45598 +0.00% BenchmarkValidateJSONSchema-4 294765 296415 +0.56% BenchmarkVersioning-4 278500 273945 -1.64% BenchmarkVirtualEndpoint-4 71906 72249 +0.48% BenchmarkApplyPolicies-4 12583 5924 -52.92% BenchmarkResponseHeaderInjection-4 399736 401316 +0.40% BenchmarkRequestIPHops-4 432 432 +0.00% BenchmarkWrappedServeHTTP-4 48400 48367 -0.07% BenchmarkCopyRequestResponse-4 35585 35585 +0.00% ``` --- api.go | 73 +++++++++++++++++++++++++++++++------ api_test.go | 4 +- auth_manager.go | 30 +-------------- coprocess.go | 14 +++---- coprocess_id_extractor.go | 23 +++--------- gateway_test.go | 1 - handler_error.go | 1 + handler_success.go | 5 ++- middleware.go | 45 ++++++++++++++++++++--- mw_api_rate_limit.go | 4 +- mw_auth_key.go | 5 +-- mw_basic_auth.go | 5 +-- mw_hmac.go | 9 ++--- mw_js_plugin.go | 14 ++++--- mw_jwt.go | 34 ++++++----------- mw_oauth2_key_exists.go | 5 +-- mw_openid.go | 7 +--- mw_organisation_activity.go | 6 ++- mw_rate_limiting.go | 11 ++---- mw_url_rewrite_test.go | 2 +- mw_virtual_endpoint.go | 10 +++-- session_manager.go | 21 +++++++---- storage/redis_cluster.go | 2 +- user/session.go | 30 ++++++--------- user/session_test.go | 23 ------------ 25 files changed, 199 insertions(+), 185 deletions(-) delete mode 100644 user/session_test.go diff --git a/api.go b/api.go index b552738de84..c0b6fcea51d 100644 --- a/api.go +++ b/api.go @@ -113,7 +113,16 @@ func applyPoliciesAndSave(keyName string, session *user.SessionState, spec *APIS mw := BaseMiddleware{ Spec: spec, } - return mw.ApplyPolicies(keyName, session) + if err := mw.ApplyPolicies(keyName, session); err != nil { + return err + } + + lifetime := session.Lifetime(spec.SessionLifetime) + if err := spec.SessionManager.UpdateSession(keyName, session, lifetime, false); err != nil { + return err + } + + return nil } func doAddOrUpdate(keyName string, newSession *user.SessionState, dontReset bool) error { @@ -307,10 +316,27 @@ func handleGetDetail(sessionKey, apiID string, byHash bool) (interface{}, int) { var session user.SessionState var ok bool session, ok = sessionManager.SessionDetail(sessionKey, byHash) + if !ok { return apiError("Key not found"), http.StatusNotFound } + quotaKey := QuotaKeyPrefix + storage.HashKey(sessionKey) + if byHash { + quotaKey = QuotaKeyPrefix + sessionKey + } + + if usedQuota, err := sessionManager.Store().GetKey(quotaKey); err == nil { + qInt, _ := strconv.Atoi(usedQuota) + remaining := session.QuotaMax - int64(qInt) + + if remaining < 0 { + session.QuotaRemaining = 0 + } else { + session.QuotaRemaining = remaining + } + } + log.WithFields(logrus.Fields{ "prefix": "api", "key": obfuscateKey(sessionKey), @@ -1513,25 +1539,50 @@ func ctxGetSession(r *http.Request) *user.SessionState { return nil } -func ctxSetSession(r *http.Request, s *user.SessionState) { +func ctxSetSession(r *http.Request, s *user.SessionState, token string, scheduleUpdate bool) { if s == nil { panic("setting a nil context SessionData") } - setCtxValue(r, SessionData, s) + + if token == "" { + token = ctxGetAuthToken(r) + } + + if s.KeyHashEmpty() { + s.SetKeyHash(storage.HashKey(token)) + } + + ctx := r.Context() + ctx = context.WithValue(ctx, SessionData, s) + ctx = context.WithValue(ctx, AuthToken, token) + + if scheduleUpdate { + ctx = context.WithValue(ctx, UpdateSession, true) + } + + setContext(r, ctx) } -func ctxGetAuthToken(r *http.Request) string { - if v := r.Context().Value(AuthHeaderValue); v != nil { - return v.(string) +func ctxScheduleSessionUpdate(r *http.Request) { + setCtxValue(r, UpdateSession, true) +} + +func ctxDisableSessionUpdate(r *http.Request) { + setCtxValue(r, UpdateSession, false) +} + +func ctxSessionUpdateScheduled(r *http.Request) bool { + if v := r.Context().Value(UpdateSession); v != nil { + return v.(bool) } - return "" + return false } -func ctxSetAuthToken(r *http.Request, t string) { - if t == "" { - panic("setting a nil context AuthHeaderValue") +func ctxGetAuthToken(r *http.Request) string { + if v := r.Context().Value(AuthToken); v != nil { + return v.(string) } - setCtxValue(r, AuthHeaderValue, t) + return "" } func ctxGetTrackedPath(r *http.Request) string { diff --git a/api_test.go b/api_test.go index 58c15ed37fd..b5d48643b93 100644 --- a/api_test.go +++ b/api_test.go @@ -759,7 +759,7 @@ func TestContextSession(t *testing.T) { if ctxGetSession(r) != nil { t.Fatal("expected ctxGetSession to return nil") } - ctxSetSession(r, &user.SessionState{}) + ctxSetSession(r, &user.SessionState{}, "", false) if ctxGetSession(r) == nil { t.Fatal("expected ctxGetSession to return non-nil") } @@ -768,7 +768,7 @@ func TestContextSession(t *testing.T) { t.Fatal("expected ctxSetSession of zero val to panic") } }() - ctxSetSession(r, nil) + ctxSetSession(r, nil, "", false) } func TestApiLoaderLongestPathFirst(t *testing.T) { diff --git a/auth_manager.go b/auth_manager.go index 60019facab6..c91a570c9e5 100644 --- a/auth_manager.go +++ b/auth_manager.go @@ -44,9 +44,8 @@ type DefaultAuthorisationManager struct { } type DefaultSessionManager struct { - store storage.Handler - asyncWrites bool - disableCacheSessionState bool + store storage.Handler + asyncWrites bool } func (b *DefaultAuthorisationManager) Init(store storage.Handler) { @@ -72,7 +71,6 @@ func (b *DefaultAuthorisationManager) KeyAuthorised(keyName string) (user.Sessio return newSession, false } - newSession.SetFirstSeenHash() return newSession, true } @@ -86,7 +84,6 @@ func (b *DefaultAuthorisationManager) KeyExpired(newSession *user.SessionState) func (b *DefaultSessionManager) Init(store storage.Handler) { b.asyncWrites = config.Global().UseAsyncSessionWrite - b.disableCacheSessionState = config.Global().LocalSessionCache.DisableCacheSessionState b.store = store b.store.Connect() } @@ -96,7 +93,6 @@ func (b *DefaultSessionManager) Store() storage.Handler { } func (b *DefaultSessionManager) ResetQuota(keyName string, session *user.SessionState) { - rawKey := QuotaKeyPrefix + storage.HashKey(keyName) log.WithFields(logrus.Fields{ "prefix": "auth-mgr", @@ -115,11 +111,6 @@ func (b *DefaultSessionManager) ResetQuota(keyName string, session *user.Session // UpdateSession updates the session state in the storage engine func (b *DefaultSessionManager) UpdateSession(keyName string, session *user.SessionState, resetTTLTo int64, hashed bool) error { - if !session.HasChanged() { - log.Debug("Session has not changed, not updating") - return nil - } - v, _ := json.Marshal(session) if hashed { @@ -128,8 +119,6 @@ func (b *DefaultSessionManager) UpdateSession(keyName string, session *user.Sess // async update and return if needed if b.asyncWrites { - b.renewSessionState(keyName, session) - if hashed { go b.store.SetRawKey(keyName, string(v), resetTTLTo) return nil @@ -147,22 +136,9 @@ func (b *DefaultSessionManager) UpdateSession(keyName string, session *user.Sess err = b.store.SetKey(keyName, string(v), resetTTLTo) } - if err == nil { - b.renewSessionState(keyName, session) - } - return err } -func (b *DefaultSessionManager) renewSessionState(keyName string, session *user.SessionState) { - // we have new session state so renew first-seen hash to prevent - session.SetFirstSeenHash() - // delete it from session cache to have it re-populated next time - if !b.disableCacheSessionState { - SessionCache.Delete(keyName) - } -} - // RemoveSession removes session from storage func (b *DefaultSessionManager) RemoveSession(keyName string, hashed bool) { if hashed { @@ -199,8 +175,6 @@ func (b *DefaultSessionManager) SessionDetail(keyName string, hashed bool) (user return session, false } - session.SetFirstSeenHash() - return session, true } diff --git a/coprocess.go b/coprocess.go index 513c1675a33..27eea41260f 100644 --- a/coprocess.go +++ b/coprocess.go @@ -328,15 +328,15 @@ func (m *CoProcessMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Requ returnedSession := TykSessionState(returnObject.Session) + // If the returned object contains metadata, add them to the session: + for k, v := range returnObject.Metadata { + returnedSession.MetaData[k] = string(v) + } + if extractor == nil { - sessionLifetime := returnedSession.Lifetime(m.Spec.SessionLifetime) - // This API is not using the ID extractor, but we've got a session: - m.Spec.SessionManager.UpdateSession(token, returnedSession, sessionLifetime, false) - ctxSetSession(r, returnedSession) - ctxSetAuthToken(r, token) + ctxSetSession(r, returnedSession, token, true) } else { - // The CP middleware did setup a session, we should pass it to the ID extractor (caching): - extractor.PostProcess(r, returnedSession, sessionID) + ctxSetSession(r, returnedSession, sessionID, true) } } diff --git a/coprocess_id_extractor.go b/coprocess_id_extractor.go index 03e2347487b..0a118aed013 100644 --- a/coprocess_id_extractor.go +++ b/coprocess_id_extractor.go @@ -15,13 +15,11 @@ import ( "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/regexp" - "github.com/TykTechnologies/tyk/user" ) // IdExtractor is the base interface for an ID extractor. type IdExtractor interface { ExtractAndCheck(*http.Request) (string, ReturnOverrides) - PostProcess(*http.Request, *user.SessionState, string) GenerateSessionID(string, BaseMiddleware) string } @@ -40,15 +38,6 @@ func (e *BaseExtractor) ExtractAndCheck(r *http.Request) (sessionID string, retu return "", ReturnOverrides{ResponseCode: 403, ResponseError: "Key not authorised"} } -// PostProcess sets context variables and updates the storage. -func (e *BaseExtractor) PostProcess(r *http.Request, session *user.SessionState, sessionID string) { - sessionLifetime := session.Lifetime(e.Spec.SessionLifetime) - e.Spec.SessionManager.UpdateSession(sessionID, session, sessionLifetime, false) - - ctxSetSession(r, session) - ctxSetAuthToken(r, sessionID) -} - // ExtractHeader is used when a HeaderSource is specified. func (e *BaseExtractor) ExtractHeader(r *http.Request) (headerValue string, err error) { headerName := e.Config.ExtractorConfig["header_name"].(string) @@ -146,11 +135,11 @@ func (e *ValueExtractor) ExtractAndCheck(r *http.Request) (sessionID string, ret sessionID = e.GenerateSessionID(extractorOutput, e.BaseMid) - previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(sessionID) + previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(sessionID, r) if keyExists { if previousSession.IdExtractorDeadline > time.Now().Unix() { - e.PostProcess(r, &previousSession, sessionID) + ctxSetSession(r, &previousSession, sessionID, true) returnOverrides = ReturnOverrides{ ResponseCode: 200, } @@ -219,11 +208,11 @@ func (e *RegexExtractor) ExtractAndCheck(r *http.Request) (SessionID string, ret } SessionID = e.GenerateSessionID(regexOutput[e.cfg.RegexMatchIndex], e.BaseMid) - previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(SessionID) + previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(SessionID, r) if keyExists { if previousSession.IdExtractorDeadline > time.Now().Unix() { - e.PostProcess(r, &previousSession, SessionID) + ctxSetSession(r, &previousSession, SessionID, true) returnOverrides = ReturnOverrides{ ResponseCode: 200, } @@ -295,10 +284,10 @@ func (e *XPathExtractor) ExtractAndCheck(r *http.Request) (SessionID string, ret SessionID = e.GenerateSessionID(output, e.BaseMid) - previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(SessionID) + previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(SessionID, r) if keyExists { if previousSession.IdExtractorDeadline > time.Now().Unix() { - e.PostProcess(r, &previousSession, SessionID) + ctxSetSession(r, &previousSession, SessionID, true) returnOverrides = ReturnOverrides{ ResponseCode: 200, } diff --git a/gateway_test.go b/gateway_test.go index 66606c990ae..00bcfe8df53 100644 --- a/gateway_test.go +++ b/gateway_test.go @@ -1231,7 +1231,6 @@ func TestKeepAliveConns(t *testing.T) { // for the API. Meaning that a single token cannot reduce service availability for other tokens by simply going over the // API's global rate limit. func TestRateLimitForAPIAndRateLimitAndQuotaCheck(t *testing.T) { - globalCfg := config.Global() globalCfg.EnableNonTransactionalRateLimiter = false globalCfg.EnableSentinelRateLImiter = true diff --git a/handler_error.go b/handler_error.go index 69ca1ba4b81..c503099a8e9 100644 --- a/handler_error.go +++ b/handler_error.go @@ -31,6 +31,7 @@ type ErrorHandler struct { // HandleError is the actual error handler and will store the error details in analytics if analytics processing is enabled. func (e *ErrorHandler) HandleError(w http.ResponseWriter, r *http.Request, errMsg string, errCode int) { + defer e.Base().UpdateRequestSession(r) var templateExtension string var contentType string diff --git a/handler_success.go b/handler_success.go index ec1eebaa99b..e42b0e2eb19 100644 --- a/handler_success.go +++ b/handler_success.go @@ -22,7 +22,9 @@ import ( // these to be implemented and is lifted pretty much from docs const ( SessionData = iota - AuthHeaderValue + UpdateSession + AuthToken + HashedAuthToken VersionData VersionDefault OrgSessionContext @@ -240,6 +242,7 @@ func recordDetail(r *http.Request, globalConf config.Config) bool { // Spec states the path is Ignored func (s *SuccessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) *http.Response { log.Debug("Started proxy") + defer s.Base().UpdateRequestSession(r) versionDef := s.Spec.VersionDefinition if versionDef.Location == "url" && versionDef.StripPath { diff --git a/middleware.go b/middleware.go index 2b324034646..cdc2eb11390 100644 --- a/middleware.go +++ b/middleware.go @@ -99,6 +99,8 @@ func createMiddleware(mw TykMiddleware) func(http.Handler) http.Handler { // No error, carry on... meta["bypass"] = "1" h.ServeHTTP(w, r) + } else { + mw.Base().UpdateRequestSession(r) } job.TimingKv("exec_time", time.Since(startTime).Nanoseconds(), meta) @@ -149,6 +151,8 @@ func (t BaseMiddleware) OrgSession(key string) (user.SessionState, bool) { log.Debug("Setting data expiry: ", session.OrgID) go t.SetOrgExpiry(session.OrgID, session.DataExpires) } + + session.SetKeyHash(storage.HashKey(key)) return session, found } @@ -168,6 +172,35 @@ func (t BaseMiddleware) OrgSessionExpiry(orgid string) int64 { return cachedVal.(int64) } +func (t BaseMiddleware) UpdateRequestSession(r *http.Request) bool { + session := ctxGetSession(r) + token := ctxGetAuthToken(r) + + if session == nil || token == "" { + return false + } + + if !ctxSessionUpdateScheduled(r) { + return false + } + + lifetime := session.Lifetime(t.Spec.SessionLifetime) + if err := t.Spec.SessionManager.UpdateSession(token, session, lifetime, false); err != nil { + log.WithError(err).Error("Can't update session") + return false + } + + // Set context state back + // Useful for benchmarks when request object stays same + ctxDisableSessionUpdate(r) + + if !t.Spec.GlobalConfig.LocalSessionCache.DisableCacheSessionState { + SessionCache.Set(session.KeyHash(), *session, cache.DefaultExpiration) + } + + return true +} + // ApplyPolicies will check if any policies are loaded. If any are, it // will overwrite the session state to use the policy values. func (t BaseMiddleware) ApplyPolicies(key string, session *user.SessionState) error { @@ -279,13 +312,13 @@ func (t BaseMiddleware) ApplyPolicies(key string, session *user.SessionState) er } session.AccessRights = rights - // Update the session in the session manager in case it gets called again - return t.Spec.SessionManager.UpdateSession(key, session, session.Lifetime(t.Spec.SessionLifetime), false) + + return nil } // CheckSessionAndIdentityForValidKey will check first the Session store for a valid key, if not found, it will try // the Auth Handler, if not found it will fail -func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string) (user.SessionState, bool) { +func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string, r *http.Request) (user.SessionState, bool) { minLength := t.Spec.GlobalConfig.MinTokenLength if minLength == 0 { // See https://github.com/TykTechnologies/tyk/issues/1681 @@ -319,6 +352,7 @@ func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string) (user.Ses log.Debug("Querying keystore") session, found := t.Spec.SessionManager.SessionDetail(key, false) if found { + session.SetKeyHash(cacheKey) // If exists, assume it has been authorized and pass on // cache it if !t.Spec.GlobalConfig.LocalSessionCache.DisableCacheSessionState { @@ -337,6 +371,7 @@ func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string) (user.Ses // 2. If not there, get it from the AuthorizationHandler session, found = t.Spec.AuthManager.KeyAuthorised(key) if found { + session.SetKeyHash(cacheKey) // If not in Session, and got it from AuthHandler, create a session with a new TTL log.Info("Recreating session for key: ", key) @@ -351,9 +386,7 @@ func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string) (user.Ses } log.Debug("Lifetime is: ", session.Lifetime(t.Spec.SessionLifetime)) - // Need to set this in order for the write to work! - session.LastUpdated = time.Now().String() - t.Spec.SessionManager.UpdateSession(key, &session, session.Lifetime(t.Spec.SessionLifetime), false) + ctxScheduleSessionUpdate(r) } return session, found diff --git a/mw_api_rate_limit.go b/mw_api_rate_limit.go index d12c1f4fd04..4642c1a36c1 100644 --- a/mw_api_rate_limit.go +++ b/mw_api_rate_limit.go @@ -8,6 +8,7 @@ import ( "time" "github.com/TykTechnologies/tyk/request" + "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/user" ) @@ -37,6 +38,7 @@ func (k *RateLimitForAPI) EnabledForSpec() bool { Per: k.Spec.GlobalRateLimit.Per, LastUpdated: strconv.Itoa(int(time.Now().UnixNano())), } + k.apiSess.SetKeyHash(storage.HashKey(k.keyName)) return true } @@ -62,7 +64,7 @@ func (k *RateLimitForAPI) handleRateLimitFailure(r *http.Request, token string) // ProcessRequest will run any checks on the request on the way through the system, return an error to have the chain fail func (k *RateLimitForAPI) ProcessRequest(w http.ResponseWriter, r *http.Request, _ interface{}) (error, int) { storeRef := k.Spec.SessionManager.Store() - reason := sessionLimiter.ForwardMessage(k.apiSess, + reason := sessionLimiter.ForwardMessage(r, k.apiSess, k.keyName, storeRef, true, diff --git a/mw_auth_key.go b/mw_auth_key.go index 283a72d1760..6392e05c9b8 100644 --- a/mw_auth_key.go +++ b/mw_auth_key.go @@ -85,7 +85,7 @@ func (k *AuthKey) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inter key = stripBearer(key) // Check if API key valid - session, keyExists := k.CheckSessionAndIdentityForValidKey(key) + session, keyExists := k.CheckSessionAndIdentityForValidKey(key, r) if !keyExists { logEntry := getLogEntryForRequest(r, key, nil) logEntry.Info("Attempted access with non-existent key.") @@ -102,8 +102,7 @@ func (k *AuthKey) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inter // Set session state on context, we will need it later switch k.Spec.BaseIdentityProvidedBy { case apidef.AuthToken, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, key) + ctxSetSession(r, &session, key, false) k.setContextVars(r, key) } diff --git a/mw_basic_auth.go b/mw_basic_auth.go index df3c6256953..7f3b5fab412 100644 --- a/mw_basic_auth.go +++ b/mw_basic_auth.go @@ -71,7 +71,7 @@ func (k *BasicAuthKeyIsValid) ProcessRequest(w http.ResponseWriter, r *http.Requ // Check if API key valid keyName := k.Spec.OrgID + authValues[0] logEntry = getLogEntryForRequest(r, keyName, nil) - session, keyExists := k.CheckSessionAndIdentityForValidKey(keyName) + session, keyExists := k.CheckSessionAndIdentityForValidKey(keyName, r) if !keyExists { logEntry.Info("Attempted access with non-existent user.") @@ -113,8 +113,7 @@ func (k *BasicAuthKeyIsValid) ProcessRequest(w http.ResponseWriter, r *http.Requ // Set session state on context, we will need it later switch k.Spec.BaseIdentityProvidedBy { case apidef.BasicAuthUser, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, keyName) + ctxSetSession(r, &session, keyName, false) } // Request is valid, carry on diff --git a/mw_hmac.go b/mw_hmac.go index 3dbe124ea4a..4d9c41fe715 100644 --- a/mw_hmac.go +++ b/mw_hmac.go @@ -73,7 +73,7 @@ func (hm *HMACMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Request, } // Get a session for the Key ID - secret, session, err := hm.getSecretAndSessionForKeyID(fieldValues.KeyID) + secret, session, err := hm.getSecretAndSessionForKeyID(r, fieldValues.KeyID) if err != nil { log.WithFields(logrus.Fields{ "prefix": "hmac", @@ -123,8 +123,7 @@ func (hm *HMACMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Request, // Set session state on context, we will need it later switch hm.Spec.BaseIdentityProvidedBy { case apidef.HMACKey, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, fieldValues.KeyID) + ctxSetSession(r, &session, fieldValues.KeyID, false) hm.setContextVars(r, fieldValues.KeyID) } @@ -222,8 +221,8 @@ type HMACFieldValues struct { Signature string } -func (hm *HMACMiddleware) getSecretAndSessionForKeyID(keyId string) (string, user.SessionState, error) { - session, keyExists := hm.CheckSessionAndIdentityForValidKey(keyId) +func (hm *HMACMiddleware) getSecretAndSessionForKeyID(r *http.Request, keyId string) (string, user.SessionState, error) { + session, keyExists := hm.CheckSessionAndIdentityForValidKey(keyId, r) if !keyExists { return "", session, errors.New("Key ID does not exist") } diff --git a/mw_js_plugin.go b/mw_js_plugin.go index c8570da6e48..422468f5fd0 100644 --- a/mw_js_plugin.go +++ b/mw_js_plugin.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "path/filepath" + "reflect" "strings" "time" @@ -136,7 +137,6 @@ func (d *DynamicMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Reques specAsJson := specToJson(d.Spec) session := new(user.SessionState) - token := ctxGetAuthToken(r) // Encode the session object (if not a pre-process) if !d.Pre && d.UseSession { @@ -249,9 +249,12 @@ func (d *DynamicMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Reques r.URL.RawQuery = values.Encode() // Save the sesison data (if modified) - if !d.Pre && d.UseSession && len(newRequestData.SessionMeta) > 0 { - session.MetaData = mapStrsToIfaces(newRequestData.SessionMeta) - d.Spec.SessionManager.UpdateSession(token, session, session.Lifetime(d.Spec.SessionLifetime), false) + if !d.Pre && d.UseSession { + newMeta := mapStrsToIfaces(newRequestData.SessionMeta) + if !reflect.DeepEqual(session.MetaData, newMeta) { + session.MetaData = newMeta + ctxScheduleSessionUpdate(r) + } } log.WithFields(logrus.Fields{ @@ -277,8 +280,7 @@ func (d *DynamicMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Reques } if d.Auth { - ctxSetSession(r, &newRequestData.Session) - ctxSetAuthToken(r, newRequestData.AuthValue) + ctxSetSession(r, &newRequestData.Session, newRequestData.AuthValue, true) } return nil, http.StatusOK diff --git a/mw_jwt.go b/mw_jwt.go index a0a6150d4ef..bf33ab0fcc4 100644 --- a/mw_jwt.go +++ b/mw_jwt.go @@ -14,7 +14,6 @@ import ( cache "github.com/pmylund/go-cache" "github.com/TykTechnologies/tyk/apidef" - "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/user" ) @@ -118,7 +117,7 @@ func (k *JWTMiddleware) getIdentityFomToken(token *jwt.Token) (string, bool) { return tykId, idFound } -func (k *JWTMiddleware) getSecret(token *jwt.Token) ([]byte, error) { +func (k *JWTMiddleware) getSecret(r *http.Request, token *jwt.Token) ([]byte, error) { config := k.Spec.APIDefinition // Check for central JWT source if config.JWTSource != "" { @@ -160,7 +159,7 @@ func (k *JWTMiddleware) getSecret(token *jwt.Token) ([]byte, error) { // Couldn't base64 decode the kid, so lets try it raw log.Debug("Getting key: ", tykId) - session, rawKeyExists := k.CheckSessionAndIdentityForValidKey(tykId) + session, rawKeyExists := k.CheckSessionAndIdentityForValidKey(tykId, r) if !rawKeyExists { log.Info("Not found!") return nil, errors.New("token invalid, key not found") @@ -178,7 +177,7 @@ func (k *JWTMiddleware) getPolicyIDFromToken(token *jwt.Token) (string, bool) { return policyID, true } -func (k *JWTMiddleware) getBasePolicyID(token *jwt.Token) (string, bool) { +func (k *JWTMiddleware) getBasePolicyID(r *http.Request, token *jwt.Token) (string, bool) { if k.Spec.JWTPolicyFieldName != "" { return k.getPolicyIDFromToken(token) } else if k.Spec.JWTClientIDBaseField != "" { @@ -189,7 +188,7 @@ func (k *JWTMiddleware) getBasePolicyID(token *jwt.Token) (string, bool) { } // Check for a regular token that matches this client ID - clientSession, exists := k.CheckSessionAndIdentityForValidKey(clientID) + clientSession, exists := k.CheckSessionAndIdentityForValidKey(clientID, r) if !exists { return "", false } @@ -229,14 +228,14 @@ func (k *JWTMiddleware) processCentralisedJWT(r *http.Request, token *jwt.Token) log.Debug("JWT Temporary session ID is: ", sessionID) - session, exists := k.CheckSessionAndIdentityForValidKey(sessionID) + session, exists := k.CheckSessionAndIdentityForValidKey(sessionID, r) if !exists { // Create it log.Debug("Key does not exist, creating") session = user.SessionState{} // We need a base policy as a template, either get it from the token itself OR a proxy client ID within Tyk - basePolicyID, foundPolicy := k.getBasePolicyID(token) + basePolicyID, foundPolicy := k.getBasePolicyID(r, token) if !foundPolicy { k.reportLoginFailure(baseFieldData, r) return errors.New("Key not authorized: no matching policy found"), http.StatusForbidden @@ -256,13 +255,11 @@ func (k *JWTMiddleware) processCentralisedJWT(r *http.Request, token *jwt.Token) session.Alias = baseFieldData // Update the session in the session manager in case it gets called again - k.Spec.SessionManager.UpdateSession(sessionID, &session, session.Lifetime(k.Spec.SessionLifetime), false) log.Debug("Policy applied to key") switch k.Spec.BaseIdentityProvidedBy { case apidef.JWTClaim, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, sessionID) + ctxSetSession(r, &session, sessionID, true) } ctxSetJWTContextVars(k.Spec, r, token) return nil, http.StatusOK @@ -304,20 +301,14 @@ func (k *JWTMiddleware) processCentralisedJWT(r *http.Request, token *jwt.Token) return errors.New("Key not authorized: could not apply new policy"), http.StatusForbidden } - cacheKey := sessionID - if k.Spec.GlobalConfig.HashKeys { - cacheKey = storage.HashStr(sessionID) - } - // update session in cache - go SessionCache.Set(cacheKey, session, cache.DefaultExpiration) + go SessionCache.Set(session.KeyHash(), session, cache.DefaultExpiration) } } log.Debug("Key found") switch k.Spec.BaseIdentityProvidedBy { case apidef.JWTClaim, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, sessionID) + ctxSetSession(r, &session, sessionID, false) } ctxSetJWTContextVars(k.Spec, r, token) return nil, http.StatusOK @@ -340,15 +331,14 @@ func (k *JWTMiddleware) processOneToOneTokenMap(r *http.Request, token *jwt.Toke } log.Debug("Using raw key ID: ", tykId) - session, exists := k.CheckSessionAndIdentityForValidKey(tykId) + session, exists := k.CheckSessionAndIdentityForValidKey(tykId, r) if !exists { k.reportLoginFailure(tykId, r) return errors.New("Key not authorized"), http.StatusForbidden } log.Debug("Raw key ID found.") - ctxSetSession(r, &session) - ctxSetAuthToken(r, tykId) + ctxSetSession(r, &session, tykId, false) ctxSetJWTContextVars(k.Spec, r, token) return nil, http.StatusOK } @@ -415,7 +405,7 @@ func (k *JWTMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Request, _ } } - val, err := k.getSecret(token) + val, err := k.getSecret(r, token) if err != nil { log.Error("Couldn't get token: ", err) return nil, err diff --git a/mw_oauth2_key_exists.go b/mw_oauth2_key_exists.go index 06b8df8aa40..6180179669f 100644 --- a/mw_oauth2_key_exists.go +++ b/mw_oauth2_key_exists.go @@ -42,7 +42,7 @@ func (k *Oauth2KeyExists) ProcessRequest(w http.ResponseWriter, r *http.Request, } accessToken := parts[1] - session, keyExists := k.CheckSessionAndIdentityForValidKey(accessToken) + session, keyExists := k.CheckSessionAndIdentityForValidKey(accessToken, r) if !keyExists { logEntry = getLogEntryForRequest(r, accessToken, nil) @@ -59,8 +59,7 @@ func (k *Oauth2KeyExists) ProcessRequest(w http.ResponseWriter, r *http.Request, // Set session state on context, we will need it later switch k.Spec.BaseIdentityProvidedBy { case apidef.OAuthKey, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, accessToken) + ctxSetSession(r, &session, accessToken, false) } // Request is valid, carry on diff --git a/mw_openid.go b/mw_openid.go index 91496e2a826..ad6383fcf60 100644 --- a/mw_openid.go +++ b/mw_openid.go @@ -170,7 +170,7 @@ func (k *OpenIDMW) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inte log.Debug("Generated Session ID: ", sessionID) - session, exists := k.CheckSessionAndIdentityForValidKey(sessionID) + session, exists := k.CheckSessionAndIdentityForValidKey(sessionID, r) if !exists { // Create it log.Debug("Key does not exist, creating") @@ -194,16 +194,13 @@ func (k *OpenIDMW) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inte session.Alias = clientID + ":" + ouser.ID // Update the session in the session manager in case it gets called again - k.Spec.SessionManager.UpdateSession(sessionID, &session, session.Lifetime(k.Spec.SessionLifetime), false) log.Debug("Policy applied to key") - } // 4. Set session state on context, we will need it later switch k.Spec.BaseIdentityProvidedBy { case apidef.OIDCUser, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, sessionID) + ctxSetSession(r, &session, sessionID, true) } ctxSetJWTContextVars(k.Spec, r, token) diff --git a/mw_organisation_activity.go b/mw_organisation_activity.go index 4eb80fffd60..72c48cd6b92 100644 --- a/mw_organisation_activity.go +++ b/mw_organisation_activity.go @@ -38,7 +38,9 @@ func (k *OrganizationMonitor) EnabledForSpec() bool { func (k *OrganizationMonitor) ProcessRequest(w http.ResponseWriter, r *http.Request, conf interface{}) (error, int) { if k.Spec.GlobalConfig.ExperimentalProcessOrgOffThread { - return k.ProcessRequestOffThread(r) + // Make a copy of request before before sending to goroutine + r2 := r.WithContext(r.Context()) + return k.ProcessRequestOffThread(r2) } return k.ProcessRequestLive(r) } @@ -62,6 +64,7 @@ func (k *OrganizationMonitor) ProcessRequestLive(r *http.Request) (error, int) { // We found a session, apply the quota and rate limiter reason := k.sessionlimiter.ForwardMessage( + r, &session, k.Spec.OrgID, k.Spec.OrgSessionManager.Store(), @@ -186,6 +189,7 @@ func (k *OrganizationMonitor) AllowAccessNext( // We found a session, apply the quota and rate limiter reason := k.sessionlimiter.ForwardMessage( + r, &session, k.Spec.OrgID, k.Spec.OrgSessionManager.Store(), diff --git a/mw_rate_limiting.go b/mw_rate_limiting.go index f50bdf587b7..09ec0076434 100644 --- a/mw_rate_limiting.go +++ b/mw_rate_limiting.go @@ -66,7 +66,9 @@ func (k *RateLimitAndQuotaCheck) ProcessRequest(w http.ResponseWriter, r *http.R token := ctxGetAuthToken(r) storeRef := k.Spec.SessionManager.Store() - reason := sessionLimiter.ForwardMessage(session, + reason := sessionLimiter.ForwardMessage( + r, + session, token, storeRef, !k.Spec.DisableRateLimit, @@ -74,13 +76,6 @@ func (k *RateLimitAndQuotaCheck) ProcessRequest(w http.ResponseWriter, r *http.R k.Spec.GlobalConfig, ) - // If either are disabled, save the write roundtrip - if !k.Spec.DisableRateLimit || !k.Spec.DisableQuota { - // Ensure quota and rate data for this session are recorded - k.Spec.SessionManager.UpdateSession(token, session, session.Lifetime(k.Spec.SessionLifetime), false) - ctxSetSession(r, session) - } - switch reason { case sessionFailNone: case sessionFailRateLimit: diff --git a/mw_url_rewrite_test.go b/mw_url_rewrite_test.go index 626ee8b6944..aa13c99422e 100644 --- a/mw_url_rewrite_test.go +++ b/mw_url_rewrite_test.go @@ -512,7 +512,7 @@ func TestRewriterTriggers(t *testing.T) { MetaData: map[string]interface{}{ "rewrite": "bar", }, - }) + }, "", false) got, err := urlRewrite(&testConf, tc.req) if err != nil { diff --git a/mw_virtual_endpoint.go b/mw_virtual_endpoint.go index 63d1f0b7d98..888e5091046 100644 --- a/mw_virtual_endpoint.go +++ b/mw_virtual_endpoint.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "net/http" "os" + "reflect" "strconv" "time" @@ -145,7 +146,6 @@ func (d *VirtualEndpoint) ServeHTTPForCache(w http.ResponseWriter, r *http.Reque specAsJson := specToJson(d.Spec) session := new(user.SessionState) - token := ctxGetAuthToken(r) // Encode the session object (if not a pre-process) if vmeta.UseSession { @@ -214,9 +214,11 @@ func (d *VirtualEndpoint) ServeHTTPForCache(w http.ResponseWriter, r *http.Reque // Save the sesison data (if modified) if vmeta.UseSession { - session.MetaData = mapStrsToIfaces(newResponseData.SessionMeta) - d.Spec.SessionManager.UpdateSession(token, session, session.Lifetime(d.Spec.SessionLifetime), false) - ctxSetSession(r, session) + newMeta := mapStrsToIfaces(newResponseData.SessionMeta) + if !reflect.DeepEqual(session.MetaData, newMeta) { + session.MetaData = newMeta + ctxSetSession(r, session, "", true) + } } log.Debug("JSVM Virtual Endpoint execution took: (ns) ", time.Now().UnixNano()-t1) diff --git a/session_manager.go b/session_manager.go index 12d69bddb27..54b94e6423d 100644 --- a/session_manager.go +++ b/session_manager.go @@ -1,6 +1,7 @@ package main import ( + "net/http" "time" "github.com/TykTechnologies/leakybucket" @@ -49,7 +50,6 @@ func (l *SessionLimiter) doRollingWindowWrite(key, rateLimiterKey, rateLimiterSe } //log.Info("break: ", (int(currentSession.Rate) - subtractor)) - if ratePerPeriodNow > int(currentSession.Rate)-subtractor { // Set a sentinel value with expire if globalConf.EnableSentinelRateLImiter { @@ -73,12 +73,12 @@ const ( // sessionFailReason if session limits have been exceeded. // Key values to manage rate are Rate and Per, e.g. Rate of 10 messages // Per 10 seconds -func (l *SessionLimiter) ForwardMessage(currentSession *user.SessionState, key string, store storage.Handler, enableRL, enableQ bool, globalConf config.Config) sessionFailReason { - rateLimiterKey := RateLimitKeyPrefix + storage.HashKey(key) - rateLimiterSentinelKey := RateLimitKeyPrefix + storage.HashKey(key) + ".BLOCKED" - +func (l *SessionLimiter) ForwardMessage(r *http.Request, currentSession *user.SessionState, key string, store storage.Handler, enableRL, enableQ bool, globalConf config.Config) sessionFailReason { if enableRL { if globalConf.EnableSentinelRateLImiter { + rateLimiterKey := RateLimitKeyPrefix + currentSession.KeyHash() + rateLimiterSentinelKey := RateLimitKeyPrefix + currentSession.KeyHash() + ".BLOCKED" + go l.doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey, currentSession, store, globalConf) // Check sentinel @@ -88,6 +88,9 @@ func (l *SessionLimiter) ForwardMessage(currentSession *user.SessionState, key s return sessionFailRateLimit } } else if globalConf.EnableRedisRollingLimiter { + rateLimiterKey := RateLimitKeyPrefix + currentSession.KeyHash() + rateLimiterSentinelKey := RateLimitKeyPrefix + currentSession.KeyHash() + ".BLOCKED" + if l.doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey, currentSession, store, globalConf) { return sessionFailRateLimit } @@ -128,7 +131,7 @@ func (l *SessionLimiter) ForwardMessage(currentSession *user.SessionState, key s currentSession.Allowance-- } - if l.RedisQuotaExceeded(currentSession, key, store) { + if l.RedisQuotaExceeded(r, currentSession, key, store) { return sessionFailQuota } } @@ -137,7 +140,7 @@ func (l *SessionLimiter) ForwardMessage(currentSession *user.SessionState, key s } -func (l *SessionLimiter) RedisQuotaExceeded(currentSession *user.SessionState, key string, store storage.Handler) bool { +func (l *SessionLimiter) RedisQuotaExceeded(r *http.Request, currentSession *user.SessionState, key string, store storage.Handler) bool { // Are they unlimited? if currentSession.QuotaMax == -1 { // No quota set @@ -146,7 +149,7 @@ func (l *SessionLimiter) RedisQuotaExceeded(currentSession *user.SessionState, k // Create the key log.Debug("[QUOTA] Inbound raw key is: ", key) - rawKey := QuotaKeyPrefix + storage.HashKey(key) + rawKey := QuotaKeyPrefix + currentSession.KeyHash() log.Debug("[QUOTA] Quota limiter key is: ", rawKey) log.Debug("Renewing with TTL: ", currentSession.QuotaRenewalRate) // INCR the key (If it equals 1 - set EXPIRE) @@ -176,6 +179,7 @@ func (l *SessionLimiter) RedisQuotaExceeded(currentSession *user.SessionState, k if qInt == 1 { current := time.Now().Unix() currentSession.QuotaRenews = current + currentSession.QuotaRenewalRate + ctxScheduleSessionUpdate(r) } // If not, pass and set the values of the session to quotamax - counter @@ -186,5 +190,6 @@ func (l *SessionLimiter) RedisQuotaExceeded(currentSession *user.SessionState, k } else { currentSession.QuotaRemaining = remaining } + return false } diff --git a/storage/redis_cluster.go b/storage/redis_cluster.go index 4562432fea4..98db67413bc 100644 --- a/storage/redis_cluster.go +++ b/storage/redis_cluster.go @@ -545,7 +545,7 @@ func (r RedisCluster) GetAndDeleteSet(keyName string) []interface{} { return nil } - log.Debug("Analytics returned: ", redVal) + log.Debug("Analytics returned: ", len(redVal)) if len(redVal) == 0 { return nil } diff --git a/user/session.go b/user/session.go index 6cc0d300808..bcd6d67eb4e 100644 --- a/user/session.go +++ b/user/session.go @@ -1,9 +1,6 @@ package user import ( - "github.com/spaolacci/murmur3" - "gopkg.in/vmihailenco/msgpack.v2" - "github.com/TykTechnologies/tyk/config" logger "github.com/TykTechnologies/tyk/log" ) @@ -72,27 +69,24 @@ type SessionState struct { IdExtractorDeadline int64 `json:"id_extractor_deadline" msg:"id_extractor_deadline"` SessionLifetime int64 `bson:"session_lifetime" json:"session_lifetime"` - firstSeenHash string + // Used to store token hash + keyHash string } -func (s *SessionState) SetFirstSeenHash() { - s.firstSeenHash = s.Hash() +func (s *SessionState) KeyHash() string { + if s.keyHash == "" { + panic("KeyHash cache not found. You should call `SetKeyHash` before.") + } + + return s.keyHash } -func (s *SessionState) Hash() string { - encoded, err := msgpack.Marshal(s) - if err != nil { - log.Error("Error encoding session data: ", err) - return "" - } - murmurHasher := murmur3.New32() - murmurHasher.Write(encoded) - murmurHasher.Sum32() - return string(murmurHasher.Sum(nil)[:]) +func (s *SessionState) SetKeyHash(hash string) { + s.keyHash = hash } -func (s *SessionState) HasChanged() bool { - return s.firstSeenHash == "" || s.firstSeenHash != s.Hash() +func (s *SessionState) KeyHashEmpty() bool { + return s.keyHash == "" } func (s *SessionState) Lifetime(fallback int64) int64 { diff --git a/user/session_test.go b/user/session_test.go deleted file mode 100644 index a4b77d44d0b..00000000000 --- a/user/session_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package user - -import ( - "testing" -) - -func BenchmarkHash(b *testing.B) { - s := SessionState{ - Allowance: 1000.0, - Rate: 1000.0, - Per: 1, - Expires: 1458669677, - QuotaRemaining: 1000, - QuotaRenewalRate: 3600, - } - - b.ReportAllocs() - for i := 0; i < b.N; i++ { - { - s.Hash() - } - } -} From 90fab7335a3eb16091b2fb4298523f4454ad33e0 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 13 Jun 2018 20:11:16 +0500 Subject: [PATCH 53/54] New token generation, support for custom hashing algorithms (#1753) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added new scheme of generating tokens, now it is JSON base64 objects, similar to how JWT works, which allows us add any meta information to the tokens. Example of new token: `eyJvcmciOiIiLCJpZCI6IjAxZGIwN2Q5NWQ0MDRjYjM5ODg3ZjUwN2ZmNTg0OGUzIiwiaCI6Im11cm11cjEyOCJ9`. After based64 decoding it turns to: `{"org":"","id":"01db07d95d404cb39887f507ff5848e3","h":"murmur128”}`. Current token has 3 fields at the moment: - “org” - organization id - “id” - token id, as we had before. For regular tokens random uuid, but for example for JWTs or OpenID based on sub. - “h” - token hashing algorithm At the moment following algorithms are supported: `murmur32`, `murmur64`, `murmur128`, `sha256`. You can set hashing algorithm to use via new config variable: `hash_key_function`. If config value not set, it generates legacy tokens like we had before. We may consider switching it to `murmur64` by default in next releases, once feature will stabilize. You can change hashing function dynamically, and have tokens with multiple hashing algos, including handling of legacy tokens. So if you turn on new key support, you old legacy tokens will continue working. Same if you decided to switch different algorithm or turn this off. Fix https://github.com/TykTechnologies/tyk/issues/1694 --- api.go | 8 +- api_test.go | 35 ++- auth_manager.go | 18 +- config/config.go | 1 + coprocess_grpc_test.go | 2 +- coprocess_id_extractor.go | 2 +- coprocess_id_extractor_test.go | 22 +- gateway_test.go | 1 + helpers_test.go | 7 +- lint/schema.go | 4 + middleware.go | 1 + multiauth_test.go | 3 +- mw_api_rate_limit.go | 2 +- mw_auth_key.go | 2 +- mw_auth_key_test.go | 87 +++++++ mw_basic_auth.go | 2 +- mw_basic_auth_test.go | 1 + mw_jwt.go | 4 +- mw_jwt_test.go | 2 +- mw_openid.go | 6 +- mw_organisation_activity.go | 4 +- mw_rate_limiting.go | 2 +- oauth_manager.go | 2 +- session_manager.go | 4 +- storage/storage.go | 88 ++++++- .../murmur3/LICENSE | 0 .../murmur3/README.md | 2 + .../murmur3/murmur.go | 3 +- .../murmur3/murmur128.go | 24 +- .../murmur3/murmur32.go | 0 .../murmur3/murmur64.go | 22 +- .../github.com/spaolacci/murmur3/.gitignore | 22 -- .../spaolacci/murmur3/murmur_test.go | 229 ------------------ vendor/vendor.json | 6 + 34 files changed, 307 insertions(+), 311 deletions(-) rename vendor/github.com/{spaolacci => TykTechnologies}/murmur3/LICENSE (100%) rename vendor/github.com/{spaolacci => TykTechnologies}/murmur3/README.md (97%) rename vendor/github.com/{spaolacci => TykTechnologies}/murmur3/murmur.go (95%) rename vendor/github.com/{spaolacci => TykTechnologies}/murmur3/murmur128.go (80%) rename vendor/github.com/{spaolacci => TykTechnologies}/murmur3/murmur32.go (100%) rename vendor/github.com/{spaolacci => TykTechnologies}/murmur3/murmur64.go (53%) delete mode 100644 vendor/github.com/spaolacci/murmur3/.gitignore delete mode 100644 vendor/github.com/spaolacci/murmur3/murmur_test.go diff --git a/api.go b/api.go index c0b6fcea51d..4c8543a295e 100644 --- a/api.go +++ b/api.go @@ -244,7 +244,7 @@ func handleAddOrUpdate(keyName string, r *http.Request) (interface{}, int) { // Only if it's NEW switch r.Method { case "POST": - keyName = newSession.OrgID + keyName + keyName = generateToken(newSession.OrgID, keyName) // It's a create, so lets hash the password setSessionPassword(&newSession) case "PUT": @@ -926,11 +926,7 @@ func createKeyHandler(w http.ResponseWriter, r *http.Request) { } if newSession.Certificate != "" { - newKey = newSession.OrgID + newSession.Certificate - - if strings.HasPrefix(newSession.Certificate, newSession.OrgID) { - newKey = newSession.Certificate - } + newKey = generateToken(newSession.OrgID, newSession.Certificate) } newSession.LastUpdated = strconv.Itoa(int(time.Now().Unix())) diff --git a/api_test.go b/api_test.go index b5d48643b93..d1bfb1f85d2 100644 --- a/api_test.go +++ b/api_test.go @@ -172,8 +172,6 @@ func TestKeyHandler(t *testing.T) { } withBadPolicyJSON, _ := json.Marshal(withBadPolicy) - knownKey := createSession() - t.Run("Create key", func(t *testing.T) { ts.Run(t, []test.TestCase{ // Master keys should be disabled by default @@ -215,6 +213,8 @@ func TestKeyHandler(t *testing.T) { }...) }) + knownKey := createSession() + t.Run("Get key", func(t *testing.T) { ts.Run(t, []test.TestCase{ {Method: "GET", Path: "/tyk/keys/unknown", AdminAuth: true, Code: 404}, @@ -257,9 +257,32 @@ func TestHashKeyHandler(t *testing.T) { // enable hashed keys listing globalConf.EnableHashedKeysListing = true config.SetGlobal(globalConf) - defer resetTestConfig() + hashTests := []struct { + hashFunction string + expectedHashSize int + desc string + }{ + {"", 8, " Legacy tokens, fallback to murmur32"}, + {storage.HashMurmur32, 8, ""}, + {storage.HashMurmur64, 16, ""}, + {storage.HashMurmur128, 32, ""}, + {storage.HashSha256, 64, ""}, + {"wrong", 16, " Should fallback to murmur64 if wrong alg"}, + } + + for _, tc := range hashTests { + globalConf.HashKeyFunction = tc.hashFunction + config.SetGlobal(globalConf) + + t.Run(fmt.Sprintf("%sHash fn: %s", tc.desc, tc.hashFunction), func(t *testing.T) { + testHashKeyHandlerHelper(t, tc.expectedHashSize) + }) + } +} + +func testHashKeyHandlerHelper(t *testing.T, expectedHashSize int) { ts := newTykTestServer() defer ts.Close() @@ -271,9 +294,13 @@ func TestHashKeyHandler(t *testing.T) { }} withAccessJSON, _ := json.Marshal(withAccess) - myKey := "my_key_id" + myKey := generateToken("", "") myKeyHash := storage.HashKey(myKey) + if len(myKeyHash) != expectedHashSize { + t.Errorf("Expected hash size: %d, got %d. Hash: %s. Key: %s", expectedHashSize, len(myKeyHash), myKeyHash, myKey) + } + t.Run("Create, get and delete key with key hashing", func(t *testing.T) { ts.Run(t, []test.TestCase{ // create key diff --git a/auth_manager.go b/auth_manager.go index c91a570c9e5..e4a0a268c01 100644 --- a/auth_manager.go +++ b/auth_manager.go @@ -185,11 +185,23 @@ func (b *DefaultSessionManager) Sessions(filter string) []string { type DefaultKeyGenerator struct{} +func generateToken(orgID, keyID string) string { + keyID = strings.TrimPrefix(keyID, orgID) + token, err := storage.GenerateToken(orgID, keyID, config.Global().HashKeyFunction) + + if err != nil { + log.WithFields(logrus.Fields{ + "prefix": "auth-mgr", + "orgID": orgID, + }).WithError(err).Warning("Issue during token generation") + } + + return token +} + // GenerateAuthKey is a utility function for generating new auth keys. Returns the storage key name and the actual key func (DefaultKeyGenerator) GenerateAuthKey(orgID string) string { - u5 := uuid.NewV4() - cleanSting := strings.Replace(u5.String(), "-", "", -1) - return orgID + cleanSting + return generateToken(orgID, "") } // GenerateHMACSecret is a utility function for generating new auth keys. Returns the storage key name and the actual key diff --git a/config/config.go b/config/config.go index faec0717c86..f7c9fbd1d4c 100644 --- a/config/config.go +++ b/config/config.go @@ -214,6 +214,7 @@ type Config struct { UseAsyncSessionWrite bool `json:"optimisations_use_async_session_write"` AllowMasterKeys bool `json:"allow_master_keys"` HashKeys bool `json:"hash_keys"` + HashKeyFunction string `json:"hash_key_function"` SuppressRedisSignalReload bool `json:"suppress_redis_signal_reload"` SupressDefaultOrgStore bool `json:"suppress_default_org_store"` UseRedisLog bool `json:"use_redis_log"` diff --git a/coprocess_grpc_test.go b/coprocess_grpc_test.go index 717568f803b..8f4847632dc 100644 --- a/coprocess_grpc_test.go +++ b/coprocess_grpc_test.go @@ -105,7 +105,7 @@ func newTestGRPCServer() (s *grpc.Server) { func loadTestGRPCAPIs() { buildAndLoadAPI(func(spec *APISpec) { spec.APIID = "1" - spec.OrgID = "default" + spec.OrgID = mockOrgID spec.Auth = apidef.Auth{ AuthHeaderName: "authorization", } diff --git a/coprocess_id_extractor.go b/coprocess_id_extractor.go index 0a118aed013..5f85fb64128 100644 --- a/coprocess_id_extractor.go +++ b/coprocess_id_extractor.go @@ -90,7 +90,7 @@ func (e *BaseExtractor) Error(r *http.Request, err error, message string) (retur func (e *BaseExtractor) GenerateSessionID(input string, mw BaseMiddleware) (sessionID string) { data := []byte(input) tokenID := fmt.Sprintf("%x", md5.Sum(data)) - sessionID = mw.Spec.OrgID + tokenID + sessionID = generateToken(mw.Spec.OrgID, tokenID) return sessionID } diff --git a/coprocess_id_extractor_test.go b/coprocess_id_extractor_test.go index 707861d208c..c23b69830eb 100644 --- a/coprocess_id_extractor_test.go +++ b/coprocess_id_extractor_test.go @@ -13,8 +13,6 @@ import ( ) const ( - extractorTestOrgID = "testorg" - extractorValueInput = "testkey" extractorRegexExpr = "prefix-(.*)" @@ -41,7 +39,7 @@ func createSpecTestFrom(t testing.TB, def *apidef.APIDefinition) *APISpec { func prepareExtractor(t testing.TB, extractorSource apidef.IdExtractorSource, extractorType apidef.IdExtractorType, config map[string]interface{}) (IdExtractor, *APISpec) { def := &apidef.APIDefinition{ - OrgID: extractorTestOrgID, + OrgID: mockOrgID, CustomMiddleware: apidef.MiddlewareSection{ IdExtractor: apidef.MiddlewareIdExtractor{ ExtractFrom: extractorSource, @@ -87,7 +85,7 @@ func prepareExtractorFormRequest(values map[string]string) *http.Request { func generateSessionID(input string) string { data := []byte(input) tokenID := fmt.Sprintf("%x", md5.Sum(data)) - return extractorTestOrgID + tokenID + return generateToken(mockOrgID, tokenID) } func TestValueExtractor(t *testing.T) { @@ -112,7 +110,7 @@ func TestValueExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -139,7 +137,7 @@ func TestValueExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -172,7 +170,7 @@ func TestRegexExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -200,7 +198,7 @@ func TestRegexExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -229,7 +227,7 @@ func TestRegexExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -261,7 +259,7 @@ func TestXPathExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -288,7 +286,7 @@ func TestXPathExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -316,7 +314,7 @@ func TestXPathExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { diff --git a/gateway_test.go b/gateway_test.go index 00bcfe8df53..c41f24289dc 100644 --- a/gateway_test.go +++ b/gateway_test.go @@ -46,6 +46,7 @@ var ( ) const defaultListenPort = 8080 +const mockOrgID = "507f1f77bcf86cd799439011" var defaultTestConfig config.Config var testServerRouter *mux.Router diff --git a/helpers_test.go b/helpers_test.go index 11f50511b9f..ca298af8584 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -29,6 +29,7 @@ import ( "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/test" "github.com/TykTechnologies/tyk/user" ) @@ -189,16 +190,16 @@ func withAuth(r *http.Request) *http.Request { // TODO: replace with /tyk/keys/create call func createSession(sGen ...func(s *user.SessionState)) string { - key := keyGen.GenerateAuthKey("") + key := generateToken("", "") session := createStandardSession() if len(sGen) > 0 { sGen[0](session) } if session.Certificate != "" { - key = session.Certificate + key = generateToken("", session.Certificate) } - FallbackKeySesionManager.UpdateSession(key, session, 60, false) + FallbackKeySesionManager.UpdateSession(storage.HashKey(key), session, 60, config.Global().HashKeys) return key } diff --git a/lint/schema.go b/lint/schema.go index 40314bd1d5c..ca662050bc9 100644 --- a/lint/schema.go +++ b/lint/schema.go @@ -288,6 +288,10 @@ const confSchema = `{ "hash_keys": { "type": "boolean" }, + "hash_key_function": { + "type": "string", + "enum": ["", "murmur32", "murmur64", "murmur128", "sha256"] + }, "health_check": { "type": ["object", "null"], "additionalProperties": false, diff --git a/middleware.go b/middleware.go index cdc2eb11390..77c7b114653 100644 --- a/middleware.go +++ b/middleware.go @@ -335,6 +335,7 @@ func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string, r *http.R if t.Spec.GlobalConfig.HashKeys { cacheKey = storage.HashStr(key) } + // Check in-memory cache if !t.Spec.GlobalConfig.LocalSessionCache.DisableCacheSessionState { cachedVal, found := SessionCache.Get(cacheKey) diff --git a/multiauth_test.go b/multiauth_test.go index 1dfb8dc4cb7..a542dea609c 100644 --- a/multiauth_test.go +++ b/multiauth_test.go @@ -99,8 +99,9 @@ func testPrepareMultiSessionBA(t testing.TB, isBench bool) (*APISpec, *http.Requ username = "0987876" } password := "TEST" + keyName := generateToken("default", username) // Basic auth sessions are stored as {org-id}{username}, so we need to append it here when we create the session. - spec.SessionManager.UpdateSession("default"+username, baSession, 60, false) + spec.SessionManager.UpdateSession(keyName, baSession, 60, false) // Create key session := createMultiAuthKeyAuthSession(isBench) diff --git a/mw_api_rate_limit.go b/mw_api_rate_limit.go index 4642c1a36c1..a312a3e177d 100644 --- a/mw_api_rate_limit.go +++ b/mw_api_rate_limit.go @@ -69,7 +69,7 @@ func (k *RateLimitForAPI) ProcessRequest(w http.ResponseWriter, r *http.Request, storeRef, true, false, - k.Spec.GlobalConfig, + &k.Spec.GlobalConfig, ) if reason == sessionFailRateLimit { diff --git a/mw_auth_key.go b/mw_auth_key.go index 6392e05c9b8..4a523ea09e8 100644 --- a/mw_auth_key.go +++ b/mw_auth_key.go @@ -70,7 +70,7 @@ func (k *AuthKey) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inter // If key not provided in header or cookie and client certificate is provided, try to find certificate based key if config.UseCertificate && key == "" && r.TLS != nil && len(r.TLS.PeerCertificates) > 0 { - key = k.Spec.OrgID + certs.HexSHA256(r.TLS.PeerCertificates[0].Raw) + key = generateToken(k.Spec.OrgID, certs.HexSHA256(r.TLS.PeerCertificates[0].Raw)) } if key == "" { diff --git a/mw_auth_key_test.go b/mw_auth_key_test.go index 804c74f11bf..4784cd27f88 100644 --- a/mw_auth_key_test.go +++ b/mw_auth_key_test.go @@ -11,9 +11,96 @@ import ( "github.com/justinas/alice" "github.com/lonelycode/go-uuid/uuid" + "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/test" "github.com/TykTechnologies/tyk/user" ) +func TestMurmur3CharBug(t *testing.T) { + defer resetTestConfig() + ts := newTykTestServer() + defer ts.Close() + + api := buildAPI(func(spec *APISpec) { + spec.UseKeylessAccess = false + spec.Proxy.ListenPath = "/" + })[0] + + genTestCase := func(key string, status int) test.TestCase { + return test.TestCase{Path: "/", Headers: map[string]string{"Authorization": key}, Code: status} + } + + t.Run("Without hashing", func(t *testing.T) { + globalConf := config.Global() + globalConf.HashKeys = false + config.SetGlobal(globalConf) + + loadAPI(api) + + key := createSession() + + ts.Run(t, []test.TestCase{ + genTestCase("wrong", 403), + genTestCase(key+"abc", 403), + genTestCase(key, 200), + }...) + }) + + t.Run("murmur32 hashing, legacy", func(t *testing.T) { + globalConf := config.Global() + globalConf.HashKeys = true + globalConf.HashKeyFunction = "" + config.SetGlobal(globalConf) + + loadAPI(api) + + key := createSession() + + ts.Run(t, []test.TestCase{ + genTestCase("wrong", 403), + // Should reject instead, just to show bug + genTestCase(key+"abc", 200), + genTestCase(key, 200), + }...) + }) + + t.Run("murmur32 hashing, json keys", func(t *testing.T) { + globalConf := config.Global() + globalConf.HashKeys = true + globalConf.HashKeyFunction = "murmur32" + config.SetGlobal(globalConf) + + loadAPI(api) + + key := createSession() + + ts.Run(t, []test.TestCase{ + genTestCase("wrong", 403), + // Should reject instead, just to show bug + genTestCase(key+"abc", 200), + genTestCase(key, 200), + }...) + }) + + t.Run("murmur64 hashing", func(t *testing.T) { + globalConf := config.Global() + globalConf.HashKeys = true + globalConf.HashKeyFunction = "murmur64" + config.SetGlobal(globalConf) + + loadAPI(api) + + key := createSession() + + ts.Run(t, []test.TestCase{ + genTestCase("wrong", 403), + // New hashing fixes the bug + genTestCase(key+"abc", 403), + genTestCase(key, 200), + }...) + }) +} + func createAuthKeyAuthSession(isBench bool) *user.SessionState { session := new(user.SessionState) // essentially non-throttled diff --git a/mw_basic_auth.go b/mw_basic_auth.go index 7f3b5fab412..8721ef33bf1 100644 --- a/mw_basic_auth.go +++ b/mw_basic_auth.go @@ -69,7 +69,7 @@ func (k *BasicAuthKeyIsValid) ProcessRequest(w http.ResponseWriter, r *http.Requ } // Check if API key valid - keyName := k.Spec.OrgID + authValues[0] + keyName := generateToken(k.Spec.OrgID, authValues[0]) logEntry = getLogEntryForRequest(r, keyName, nil) session, keyExists := k.CheckSessionAndIdentityForValidKey(keyName, r) if !keyExists { diff --git a/mw_basic_auth_test.go b/mw_basic_auth_test.go index 6923e1c8305..7c5b0230950 100644 --- a/mw_basic_auth_test.go +++ b/mw_basic_auth_test.go @@ -20,6 +20,7 @@ func testPrepareBasicAuth() *user.SessionState { session := createStandardSession() session.BasicAuthData.Password = "password" session.AccessRights = map[string]user.AccessDefinition{"test": {APIID: "test", Versions: []string{"v1"}}} + session.OrgID = "default" buildAndLoadAPI(func(spec *APISpec) { spec.UseBasicAuth = true diff --git a/mw_jwt.go b/mw_jwt.go index bf33ab0fcc4..0961ecabb0a 100644 --- a/mw_jwt.go +++ b/mw_jwt.go @@ -223,8 +223,8 @@ func (k *JWTMiddleware) processCentralisedJWT(r *http.Request, token *jwt.Token) } log.Debug("Base Field ID set to: ", baseFieldData) data := []byte(baseFieldData) - tokenID := fmt.Sprintf("%x", md5.Sum(data)) - sessionID := k.Spec.OrgID + tokenID + keyID := fmt.Sprintf("%x", md5.Sum(data)) + sessionID := generateToken(k.Spec.OrgID, keyID) log.Debug("JWT Temporary session ID is: ", sessionID) diff --git a/mw_jwt_test.go b/mw_jwt_test.go index 8f4acf173b1..8a7e04367ef 100644 --- a/mw_jwt_test.go +++ b/mw_jwt_test.go @@ -643,7 +643,7 @@ func TestJWTExistingSessionRSAWithRawSourcePolicyIDChanged(t *testing.T) { t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() }) - sessionID := fmt.Sprintf("%x", md5.Sum([]byte("user"))) + sessionID := generateToken("", fmt.Sprintf("%x", md5.Sum([]byte("user")))) authHeaders := map[string]string{"authorization": jwtToken} t.Run("Initial request with 1st policy", func(t *testing.T) { diff --git a/mw_openid.go b/mw_openid.go index ad6383fcf60..7eeafbe6e4c 100644 --- a/mw_openid.go +++ b/mw_openid.go @@ -160,12 +160,12 @@ func (k *OpenIDMW) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inte } data := []byte(ouser.ID) - tokenID := fmt.Sprintf("%x", md5.Sum(data)) - sessionID := k.Spec.OrgID + tokenID + keyID := fmt.Sprintf("%x", md5.Sum(data)) + sessionID := generateToken(k.Spec.OrgID, keyID) if k.Spec.OpenIDOptions.SegregateByClient { // We are segregating by client, so use it as part of the internal token log.Debug("Client ID:", clientID) - sessionID = k.Spec.OrgID + fmt.Sprintf("%x", md5.Sum([]byte(clientID))) + tokenID + sessionID = generateToken(k.Spec.OrgID, fmt.Sprintf("%x", md5.Sum([]byte(clientID)))+keyID) } log.Debug("Generated Session ID: ", sessionID) diff --git a/mw_organisation_activity.go b/mw_organisation_activity.go index 72c48cd6b92..8b58323f5f6 100644 --- a/mw_organisation_activity.go +++ b/mw_organisation_activity.go @@ -70,7 +70,7 @@ func (k *OrganizationMonitor) ProcessRequestLive(r *http.Request) (error, int) { k.Spec.OrgSessionManager.Store(), session.Per > 0 && session.Rate > 0, true, - k.Spec.GlobalConfig, + &k.Spec.GlobalConfig, ) k.Spec.OrgSessionManager.UpdateSession(k.Spec.OrgID, &session, session.Lifetime(k.Spec.SessionLifetime), false) @@ -195,7 +195,7 @@ func (k *OrganizationMonitor) AllowAccessNext( k.Spec.OrgSessionManager.Store(), session.Per > 0 && session.Rate > 0, true, - k.Spec.GlobalConfig, + &k.Spec.GlobalConfig, ) k.Spec.OrgSessionManager.UpdateSession(k.Spec.OrgID, &session, session.Lifetime(k.Spec.SessionLifetime), false) diff --git a/mw_rate_limiting.go b/mw_rate_limiting.go index 09ec0076434..99e204e3ef2 100644 --- a/mw_rate_limiting.go +++ b/mw_rate_limiting.go @@ -73,7 +73,7 @@ func (k *RateLimitAndQuotaCheck) ProcessRequest(w http.ResponseWriter, r *http.R storeRef, !k.Spec.DisableRateLimit, !k.Spec.DisableQuota, - k.Spec.GlobalConfig, + &k.Spec.GlobalConfig, ) switch reason { diff --git a/oauth_manager.go b/oauth_manager.go index 5972c0ee6ec..865ee2b6049 100644 --- a/oauth_manager.go +++ b/oauth_manager.go @@ -321,7 +321,7 @@ func (o *OAuthManager) HandleAccess(r *http.Request) *osin.Response { log.Debug("New token: ", new_token.(string)) log.Debug("Keys: ", session.OauthKeys) - keyName := o.API.OrgID + username + keyName := generateToken(o.API.OrgID, username) log.Debug("Updating user:", keyName) err := o.API.SessionManager.UpdateSession(keyName, session, session.Lifetime(o.API.SessionLifetime), false) diff --git a/session_manager.go b/session_manager.go index 54b94e6423d..e10d2eefc1c 100644 --- a/session_manager.go +++ b/session_manager.go @@ -34,7 +34,7 @@ type SessionLimiter struct { bucketStore leakybucket.Storage } -func (l *SessionLimiter) doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey string, currentSession *user.SessionState, store storage.Handler, globalConf config.Config) bool { +func (l *SessionLimiter) doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey string, currentSession *user.SessionState, store storage.Handler, globalConf *config.Config) bool { log.Debug("[RATELIMIT] Inbound raw key is: ", key) log.Debug("[RATELIMIT] Rate limiter key is: ", rateLimiterKey) pipeline := globalConf.EnableNonTransactionalRateLimiter @@ -73,7 +73,7 @@ const ( // sessionFailReason if session limits have been exceeded. // Key values to manage rate are Rate and Per, e.g. Rate of 10 messages // Per 10 seconds -func (l *SessionLimiter) ForwardMessage(r *http.Request, currentSession *user.SessionState, key string, store storage.Handler, enableRL, enableQ bool, globalConf config.Config) sessionFailReason { +func (l *SessionLimiter) ForwardMessage(r *http.Request, currentSession *user.SessionState, key string, store storage.Handler, enableRL, enableQ bool, globalConf *config.Config) sessionFailReason { if enableRL { if globalConf.EnableSentinelRateLImiter { rateLimiterKey := RateLimitKeyPrefix + currentSession.KeyHash() diff --git a/storage/storage.go b/storage/storage.go index e5c6ac566ce..f1c135dc096 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1,11 +1,18 @@ package storage import ( + "crypto/sha256" + "encoding/base64" "encoding/hex" "errors" + "fmt" + "hash" + "strings" - "github.com/spaolacci/murmur3" + "github.com/buger/jsonparser" + "github.com/satori/go.uuid" + "github.com/TykTechnologies/murmur3" "github.com/TykTechnologies/tyk/config" logger "github.com/TykTechnologies/tyk/log" ) @@ -47,8 +54,85 @@ type Handler interface { RemoveSortedSetRange(string, string, string) error } +const defaultHashAlgorithm = "murmur64" + +// If hashing algorithm is empty, use legacy key generation +func GenerateToken(orgID, keyID, hashAlgorithm string) (string, error) { + if keyID == "" { + keyID = strings.Replace(uuid.NewV4().String(), "-", "", -1) + } + + if hashAlgorithm != "" { + _, err := hashFunction(hashAlgorithm) + if err != nil { + hashAlgorithm = defaultHashAlgorithm + } + + jsonToken := fmt.Sprintf(`{"org":"%s","id":"%s","h":"%s"}`, orgID, keyID, hashAlgorithm) + return base64.StdEncoding.EncodeToString([]byte(jsonToken)), err + } + + // Legacy keys + return orgID + keyID, nil +} + +// `{"` in base64 +const B64JSONPrefix = "ey" + +func TokenHashAlgo(token string) string { + // Legacy tokens not b64 and not JSON records + if strings.HasPrefix(token, B64JSONPrefix) { + if jsonToken, err := base64.StdEncoding.DecodeString(token); err == nil { + hashAlgo, _ := jsonparser.GetString(jsonToken, "h") + return hashAlgo + } + } + + return "" +} + +func TokenOrg(token string) string { + if strings.HasPrefix(token, B64JSONPrefix) { + if jsonToken, err := base64.StdEncoding.DecodeString(token); err == nil { + // Checking error in case if it is a legacy tooken which just by accided has the same b64JSON prefix + if org, err := jsonparser.GetString(jsonToken, "org"); err == nil { + return org + } + } + } + + // 24 is mongo bson id length + if len(token) > 24 { + return token[:24] + } + + return "" +} + +var ( + HashSha256 = "sha256" + HashMurmur32 = "murmur32" + HashMurmur64 = "murmur64" + HashMurmur128 = "murmur128" +) + +func hashFunction(algorithm string) (hash.Hash, error) { + switch algorithm { + case HashSha256: + return sha256.New(), nil + case HashMurmur64: + return murmur3.New64(), nil + case HashMurmur128: + return murmur3.New128(), nil + case "", HashMurmur32: + return murmur3.New32(), nil + default: + return murmur3.New32(), fmt.Errorf("Unknown key hash function: %s. Falling back to murmur32.", algorithm) + } +} + func HashStr(in string) string { - h := murmur3.New32() + h, _ := hashFunction(TokenHashAlgo(in)) h.Write([]byte(in)) return hex.EncodeToString(h.Sum(nil)) } diff --git a/vendor/github.com/spaolacci/murmur3/LICENSE b/vendor/github.com/TykTechnologies/murmur3/LICENSE similarity index 100% rename from vendor/github.com/spaolacci/murmur3/LICENSE rename to vendor/github.com/TykTechnologies/murmur3/LICENSE diff --git a/vendor/github.com/spaolacci/murmur3/README.md b/vendor/github.com/TykTechnologies/murmur3/README.md similarity index 97% rename from vendor/github.com/spaolacci/murmur3/README.md rename to vendor/github.com/TykTechnologies/murmur3/README.md index 1edf62300d2..e463678a05e 100644 --- a/vendor/github.com/spaolacci/murmur3/README.md +++ b/vendor/github.com/TykTechnologies/murmur3/README.md @@ -1,6 +1,8 @@ murmur3 ======= +[![Build Status](https://travis-ci.org/spaolacci/murmur3.svg?branch=master)](https://travis-ci.org/spaolacci/murmur3) + Native Go implementation of Austin Appleby's third MurmurHash revision (aka MurmurHash3). diff --git a/vendor/github.com/spaolacci/murmur3/murmur.go b/vendor/github.com/TykTechnologies/murmur3/murmur.go similarity index 95% rename from vendor/github.com/spaolacci/murmur3/murmur.go rename to vendor/github.com/TykTechnologies/murmur3/murmur.go index f99557cc3ec..1252cf73a79 100644 --- a/vendor/github.com/spaolacci/murmur3/murmur.go +++ b/vendor/github.com/TykTechnologies/murmur3/murmur.go @@ -3,8 +3,6 @@ // license that can be found in the LICENSE file. /* -Native (and fast) implementation of Austin Appleby's MurmurHash3. - Package murmur3 implements Austin Appleby's non-cryptographic MurmurHash3. Reference implementation: @@ -26,6 +24,7 @@ type digest struct { clen int // Digested input cumulative length. tail []byte // 0 to Size()-1 bytes view of `buf'. buf [16]byte // Expected (but not required) to be Size() large. + seed uint32 // Seed for initializing the hash. bmixer } diff --git a/vendor/github.com/spaolacci/murmur3/murmur128.go b/vendor/github.com/TykTechnologies/murmur3/murmur128.go similarity index 80% rename from vendor/github.com/spaolacci/murmur3/murmur128.go rename to vendor/github.com/TykTechnologies/murmur3/murmur128.go index 16c34d6fbc8..a4b618b5f3d 100644 --- a/vendor/github.com/spaolacci/murmur3/murmur128.go +++ b/vendor/github.com/TykTechnologies/murmur3/murmur128.go @@ -18,6 +18,7 @@ var ( _ bmixer = new(digest128) ) +// Hash128 represents a 128-bit hasher // Hack: the standard api doesn't define any Hash128 interface. type Hash128 interface { hash.Hash @@ -31,8 +32,13 @@ type digest128 struct { h2 uint64 // Unfinalized running hash part 2. } -func New128() Hash128 { +// New128 returns a 128-bit hasher +func New128() Hash128 { return New128WithSeed(0) } + +// New128WithSeed returns a 128-bit hasher set with explicit seed value +func New128WithSeed(seed uint32) Hash128 { d := new(digest128) + d.seed = seed d.bmixer = d d.Reset() return d @@ -40,10 +46,10 @@ func New128() Hash128 { func (d *digest128) Size() int { return 16 } -func (d *digest128) reset() { d.h1, d.h2 = 0, 0 } +func (d *digest128) reset() { d.h1, d.h2 = uint64(d.seed), uint64(d.seed) } func (d *digest128) Sum(b []byte) []byte { - h1, h2 := d.h1, d.h2 + h1, h2 := d.Sum128() return append(b, byte(h1>>56), byte(h1>>48), byte(h1>>40), byte(h1>>32), byte(h1>>24), byte(h1>>16), byte(h1>>8), byte(h1), @@ -181,8 +187,16 @@ func rotl64(x uint64, r byte) uint64 { // hasher := New128() // hasher.Write(data) // return hasher.Sum128() -func Sum128(data []byte) (h1 uint64, h2 uint64) { - d := &digest128{h1: 0, h2: 0} +func Sum128(data []byte) (h1 uint64, h2 uint64) { return Sum128WithSeed(data, 0) } + +// Sum128WithSeed returns the MurmurHash3 sum of data. It is equivalent to the +// following sequence (without the extra burden and the extra allocation): +// hasher := New128WithSeed(seed) +// hasher.Write(data) +// return hasher.Sum128() +func Sum128WithSeed(data []byte, seed uint32) (h1 uint64, h2 uint64) { + d := &digest128{h1: uint64(seed), h2: uint64(seed)} + d.seed = seed d.tail = d.bmix(data) d.clen = len(data) return d.Sum128() diff --git a/vendor/github.com/spaolacci/murmur3/murmur32.go b/vendor/github.com/TykTechnologies/murmur3/murmur32.go similarity index 100% rename from vendor/github.com/spaolacci/murmur3/murmur32.go rename to vendor/github.com/TykTechnologies/murmur3/murmur32.go diff --git a/vendor/github.com/spaolacci/murmur3/murmur64.go b/vendor/github.com/TykTechnologies/murmur3/murmur64.go similarity index 53% rename from vendor/github.com/spaolacci/murmur3/murmur64.go rename to vendor/github.com/TykTechnologies/murmur3/murmur64.go index fdd4398e398..65a410ae0b9 100644 --- a/vendor/github.com/spaolacci/murmur3/murmur64.go +++ b/vendor/github.com/TykTechnologies/murmur3/murmur64.go @@ -14,13 +14,17 @@ var ( // digest64 is half a digest128. type digest64 digest128 -func New64() hash.Hash64 { - d := (*digest64)(New128().(*digest128)) +// New64 returns a 64-bit hasher +func New64() hash.Hash64 { return New64WithSeed(0) } + +// New64WithSeed returns a 64-bit hasher set with explicit seed value +func New64WithSeed(seed uint32) hash.Hash64 { + d := (*digest64)(New128WithSeed(seed).(*digest128)) return d } func (d *digest64) Sum(b []byte) []byte { - h1 := d.h1 + h1 := d.Sum64() return append(b, byte(h1>>56), byte(h1>>48), byte(h1>>40), byte(h1>>32), byte(h1>>24), byte(h1>>16), byte(h1>>8), byte(h1)) @@ -36,8 +40,16 @@ func (d *digest64) Sum64() uint64 { // hasher := New64() // hasher.Write(data) // return hasher.Sum64() -func Sum64(data []byte) uint64 { - d := &digest128{h1: 0, h2: 0} +func Sum64(data []byte) uint64 { return Sum64WithSeed(data, 0) } + +// Sum64WithSeed returns the MurmurHash3 sum of data. It is equivalent to the +// following sequence (without the extra burden and the extra allocation): +// hasher := New64WithSeed(seed) +// hasher.Write(data) +// return hasher.Sum64() +func Sum64WithSeed(data []byte, seed uint32) uint64 { + d := &digest128{h1: uint64(seed), h2: uint64(seed)} + d.seed = seed d.tail = d.bmix(data) d.clen = len(data) h1, _ := d.Sum128() diff --git a/vendor/github.com/spaolacci/murmur3/.gitignore b/vendor/github.com/spaolacci/murmur3/.gitignore deleted file mode 100644 index 00268614f04..00000000000 --- a/vendor/github.com/spaolacci/murmur3/.gitignore +++ /dev/null @@ -1,22 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe diff --git a/vendor/github.com/spaolacci/murmur3/murmur_test.go b/vendor/github.com/spaolacci/murmur3/murmur_test.go deleted file mode 100644 index 32789bdb4fa..00000000000 --- a/vendor/github.com/spaolacci/murmur3/murmur_test.go +++ /dev/null @@ -1,229 +0,0 @@ -package murmur3 - -import ( - "hash" - "testing" -) - -var data = []struct { - h32 uint32 - h64_1 uint64 - h64_2 uint64 - s string -}{ - {0x00000000, 0x0000000000000000, 0x0000000000000000, ""}, - {0x248bfa47, 0xcbd8a7b341bd9b02, 0x5b1e906a48ae1d19, "hello"}, - {0x149bbb7f, 0x342fac623a5ebc8e, 0x4cdcbc079642414d, "hello, world"}, - {0xe31e8a70, 0xb89e5988b737affc, 0x664fc2950231b2cb, "19 Jan 2038 at 3:14:07 AM"}, - {0xd5c48bfc, 0xcd99481f9ee902c9, 0x695da1a38987b6e7, "The quick brown fox jumps over the lazy dog."}, -} - -func TestRef(t *testing.T) { - for _, elem := range data { - - var h32 hash.Hash32 = New32() - h32.Write([]byte(elem.s)) - if v := h32.Sum32(); v != elem.h32 { - t.Errorf("'%s': 0x%x (want 0x%x)", elem.s, v, elem.h32) - } - - if v := Sum32([]byte(elem.s)); v != elem.h32 { - t.Errorf("'%s': 0x%x (want 0x%x)", elem.s, v, elem.h32) - } - - var h64 hash.Hash64 = New64() - h64.Write([]byte(elem.s)) - if v := h64.Sum64(); v != elem.h64_1 { - t.Errorf("'%s': 0x%x (want 0x%x)", elem.s, v, elem.h64_1) - } - - if v := Sum64([]byte(elem.s)); v != elem.h64_1 { - t.Errorf("'%s': 0x%x (want 0x%x)", elem.s, v, elem.h64_1) - } - - var h128 Hash128 = New128() - h128.Write([]byte(elem.s)) - if v1, v2 := h128.Sum128(); v1 != elem.h64_1 || v2 != elem.h64_2 { - t.Errorf("'%s': 0x%x-0x%x (want 0x%x-0x%x)", elem.s, v1, v2, elem.h64_1, elem.h64_2) - } - - if v1, v2 := Sum128([]byte(elem.s)); v1 != elem.h64_1 || v2 != elem.h64_2 { - t.Errorf("'%s': 0x%x-0x%x (want 0x%x-0x%x)", elem.s, v1, v2, elem.h64_1, elem.h64_2) - } - } -} - -func TestIncremental(t *testing.T) { - for _, elem := range data { - h32 := New32() - h128 := New128() - for i, j, k := 0, 0, len(elem.s); i < k; i = j { - j = 2*i + 3 - if j > k { - j = k - } - s := elem.s[i:j] - print(s + "|") - h32.Write([]byte(s)) - h128.Write([]byte(s)) - } - println() - if v := h32.Sum32(); v != elem.h32 { - t.Errorf("'%s': 0x%x (want 0x%x)", elem.s, v, elem.h32) - } - if v1, v2 := h128.Sum128(); v1 != elem.h64_1 || v2 != elem.h64_2 { - t.Errorf("'%s': 0x%x-0x%x (want 0x%x-0x%x)", elem.s, v1, v2, elem.h64_1, elem.h64_2) - } - } -} - -//--- - -func bench32(b *testing.B, length int) { - buf := make([]byte, length) - b.SetBytes(int64(length)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - Sum32(buf) - } -} - -func Benchmark32_1(b *testing.B) { - bench32(b, 1) -} -func Benchmark32_2(b *testing.B) { - bench32(b, 2) -} -func Benchmark32_4(b *testing.B) { - bench32(b, 4) -} -func Benchmark32_8(b *testing.B) { - bench32(b, 8) -} -func Benchmark32_16(b *testing.B) { - bench32(b, 16) -} -func Benchmark32_32(b *testing.B) { - bench32(b, 32) -} -func Benchmark32_64(b *testing.B) { - bench32(b, 64) -} -func Benchmark32_128(b *testing.B) { - bench32(b, 128) -} -func Benchmark32_256(b *testing.B) { - bench32(b, 256) -} -func Benchmark32_512(b *testing.B) { - bench32(b, 512) -} -func Benchmark32_1024(b *testing.B) { - bench32(b, 1024) -} -func Benchmark32_2048(b *testing.B) { - bench32(b, 2048) -} -func Benchmark32_4096(b *testing.B) { - bench32(b, 4096) -} -func Benchmark32_8192(b *testing.B) { - bench32(b, 8192) -} - -//--- - -func benchPartial32(b *testing.B, length int) { - buf := make([]byte, length) - b.SetBytes(int64(length)) - - start := (32 / 8) / 2 - chunks := 7 - k := length / chunks - tail := (length - start) % k - - b.ResetTimer() - for i := 0; i < b.N; i++ { - hasher := New32() - hasher.Write(buf[0:start]) - - for j := start; j+k <= length; j += k { - hasher.Write(buf[j : j+k]) - } - - hasher.Write(buf[length-tail:]) - hasher.Sum32() - } -} - -func BenchmarkPartial32_8(b *testing.B) { - benchPartial32(b, 8) -} -func BenchmarkPartial32_16(b *testing.B) { - benchPartial32(b, 16) -} -func BenchmarkPartial32_32(b *testing.B) { - benchPartial32(b, 32) -} -func BenchmarkPartial32_64(b *testing.B) { - benchPartial32(b, 64) -} -func BenchmarkPartial32_128(b *testing.B) { - benchPartial32(b, 128) -} - -//--- - -func bench128(b *testing.B, length int) { - buf := make([]byte, length) - b.SetBytes(int64(length)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - Sum128(buf) - } -} - -func Benchmark128_1(b *testing.B) { - bench128(b, 1) -} -func Benchmark128_2(b *testing.B) { - bench128(b, 2) -} -func Benchmark128_4(b *testing.B) { - bench128(b, 4) -} -func Benchmark128_8(b *testing.B) { - bench128(b, 8) -} -func Benchmark128_16(b *testing.B) { - bench128(b, 16) -} -func Benchmark128_32(b *testing.B) { - bench128(b, 32) -} -func Benchmark128_64(b *testing.B) { - bench128(b, 64) -} -func Benchmark128_128(b *testing.B) { - bench128(b, 128) -} -func Benchmark128_256(b *testing.B) { - bench128(b, 256) -} -func Benchmark128_512(b *testing.B) { - bench128(b, 512) -} -func Benchmark128_1024(b *testing.B) { - bench128(b, 1024) -} -func Benchmark128_2048(b *testing.B) { - bench128(b, 2048) -} -func Benchmark128_4096(b *testing.B) { - bench128(b, 4096) -} -func Benchmark128_8192(b *testing.B) { - bench128(b, 8192) -} - -//--- diff --git a/vendor/vendor.json b/vendor/vendor.json index 6cca4a0746a..ef1f1549055 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -78,6 +78,12 @@ "revision": "55ff0f4b9b3de929d99bad783723811ff7add9b6", "revisionTime": "2016-12-01T17:12:39Z" }, + { + "checksumSHA1": "FfOYh5n7gVJDqDcBbxySdCzzs1Y=", + "path": "github.com/TykTechnologies/murmur3", + "revision": "1915e687e465f5085f46a3a1339fb2d004e29dfb", + "revisionTime": "2018-06-02T12:20:59Z" + }, { "checksumSHA1": "v64AG0uGiKryWV/kqq5360zw1ts=", "path": "github.com/TykTechnologies/openid2go/openid", From fd3af4861451e37d676cf8ef14024882323fb737 Mon Sep 17 00:00:00 2001 From: Matias Insaurralde Date: Mon, 28 May 2018 11:51:20 -0400 Subject: [PATCH 54/54] Add a wrapper for command-line parameters using kingpin.Application Reorganize the lint command as a kingpin.Command (avoid using it as a top-level argument) Modified the importer code to use boolean flags and moved it to its own package --- cli/cli.go | 91 ++++++++++++ cli/importer/importer.go | 247 ++++++++++++++++++++++++++++++++ {lint => cli/lint}/lint.go | 0 {lint => cli/lint}/lint_test.go | 2 +- {lint => cli/lint}/schema.go | 0 command_mode.go | 204 -------------------------- gateway_test.go | 9 +- instrumentation_handlers.go | 3 +- main.go | 92 +++--------- 9 files changed, 364 insertions(+), 284 deletions(-) create mode 100644 cli/cli.go create mode 100644 cli/importer/importer.go rename {lint => cli/lint}/lint.go (100%) rename {lint => cli/lint}/lint_test.go (99%) rename {lint => cli/lint}/schema.go (100%) delete mode 100644 command_mode.go diff --git a/cli/cli.go b/cli/cli.go new file mode 100644 index 00000000000..484b59ed956 --- /dev/null +++ b/cli/cli.go @@ -0,0 +1,91 @@ +package cli + +import ( + "fmt" + "os" + + kingpin "gopkg.in/alecthomas/kingpin.v2" + + "github.com/TykTechnologies/tyk/cli/importer" + "github.com/TykTechnologies/tyk/cli/lint" +) + +const ( + appName = "tyk" + appDesc = "Tyk Gateway" +) + +var ( + // Conf specifies the configuration file path. + Conf *string + // Port specifies the listen port. + Port *string + // MemProfile enables memory profiling. + MemProfile *bool + // CPUProfile enables CPU profiling. + CPUProfile *bool + // BlockProfile enables block profiling. + BlockProfile *bool + // MutexProfile enables block profiling. + MutexProfile *bool + // HTTPProfile exposes a HTTP endpoint for accessing profiling data. + HTTPProfile *bool + // DebugMode sets the log level to debug mode. + DebugMode *bool + // LogInstrumentation outputs instrumentation data to stdout. + LogInstrumentation *bool + + app *kingpin.Application +) + +// Init sets all flags and subcommands. +func Init(version string, confPaths []string) { + app = kingpin.New(appName, appDesc) + app.HelpFlag.Short('h') + app.Version(version) + + // Start/default command: + startCmd := app.Command("start", "Starts the Tyk Gateway") + Conf = startCmd.Flag("conf", "load a named configuration file").PlaceHolder("FILE").String() + Port = startCmd.Flag("port", "listen on PORT (overrides config file)").String() + MemProfile = startCmd.Flag("memprofile", "generate a memory profile").Bool() + CPUProfile = startCmd.Flag("cpuprofile", "generate a cpu profile").Bool() + BlockProfile = startCmd.Flag("blockprofile", "generate a block profile").Bool() + MutexProfile = startCmd.Flag("mutexprofile", "generate a mutex profile").Bool() + HTTPProfile = startCmd.Flag("httpprofile", "expose runtime profiling data via HTTP").Bool() + DebugMode = startCmd.Flag("debug", "enable debug mode").Bool() + LogInstrumentation = startCmd.Flag("log-intrumentation", "output intrumentation output to stdout").Bool() + + startCmd.Action(func(ctx *kingpin.ParseContext) error { + return nil + }) + startCmd.Default() + + // Linter: + lintCmd := app.Command("lint", "Runs a linter on Tyk configuration file") + lintCmd.Action(func(c *kingpin.ParseContext) error { + path, lines, err := lint.Run(confPaths) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if len(lines) == 0 { + fmt.Printf("found no issues in %s\n", path) + return nil + } + fmt.Printf("issues found in %s:\n", path) + for _, line := range lines { + fmt.Println(line) + } + os.Exit(1) + return nil + }) + + // Import command: + importer.AddTo(app) +} + +// Parse parses the command-line arguments. +func Parse() { + kingpin.MustParse(app.Parse(os.Args[1:])) +} diff --git a/cli/importer/importer.go b/cli/importer/importer.go new file mode 100644 index 00000000000..ffd908be314 --- /dev/null +++ b/cli/importer/importer.go @@ -0,0 +1,247 @@ +package importer + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + log "github.com/Sirupsen/logrus" + + kingpin "gopkg.in/alecthomas/kingpin.v2" + + "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/apidef/importer" +) + +const ( + cmdName = "import" + cmdDesc = "Imports a BluePrint/Swagger file" +) + +var ( + imp *Importer + errUnknownMode = errors.New("Unknown mode") +) + +// Importer wraps the import functionality. +type Importer struct { + input *string + swaggerMode *bool + bluePrintMode *bool + createAPI *bool + orgID *string + upstreamTarget *string + asMock *bool + forAPI *string + asVersion *string +} + +func init() { + imp = &Importer{} +} + +// AddTo initializes an importer object. +func AddTo(app *kingpin.Application) { + cmd := app.Command(cmdName, cmdDesc) + imp.input = cmd.Arg("input file", "e.g. blueprint.json, swagger.json, etc.").String() + imp.swaggerMode = cmd.Flag("swagger", "Use Swagger mode").Bool() + imp.bluePrintMode = cmd.Flag("blueprint", "Use BluePrint mode").Bool() + imp.createAPI = cmd.Flag("create-api", "Creates a new API definition from the blueprint").Bool() + imp.orgID = cmd.Flag("org-id", "assign the API Definition to this org_id (required with create-api").String() + imp.upstreamTarget = cmd.Flag("upstream-target", "set the upstream target for the definition").PlaceHolder("URL").String() + imp.asMock = cmd.Flag("as-mock", "creates the API as a mock based on example fields").Bool() + imp.forAPI = cmd.Flag("for-api", "adds blueprint to existing API Definition as version").PlaceHolder("PATH").String() + imp.asVersion = cmd.Flag("as-version", "the version number to use when inserting").PlaceHolder("VERSION").String() + cmd.Action(imp.Import) +} + +// Import performs the import process. +func (i *Importer) Import(ctx *kingpin.ParseContext) (err error) { + if *i.swaggerMode { + err = i.handleSwaggerMode() + if err != nil { + log.Fatal(err) + os.Exit(1) + } + } else if *i.bluePrintMode { + err = i.handleBluePrintMode() + if err != nil { + log.Fatal(err) + os.Exit(1) + } + } else { + log.Fatal(errUnknownMode) + os.Exit(1) + } + os.Exit(0) + return nil +} + +func (i *Importer) handleBluePrintMode() error { + if !*i.createAPI { + // Different branch, here we need an API Definition to modify + if *i.forAPI == "" { + return fmt.Errorf("If adding to an API, the path to the definition must be listed") + } + + if *i.asVersion == "" { + return fmt.Errorf("No version defined for this import operation, please set an import ID using the --as-version flag") + } + + defFromFile, err := i.apiDefLoadFile(*i.forAPI) + if err != nil { + return fmt.Errorf("failed to load and decode file data for API Definition: %v", err) + } + + bp, err := i.bluePrintLoadFile(*i.input) + if err != nil { + return fmt.Errorf("File load error: %v", err) + } + versionData, err := bp.ConvertIntoApiVersion(*i.asMock) + if err != nil { + return fmt.Errorf("onversion into API Def failed: %v", err) + } + + if err := bp.InsertIntoAPIDefinitionAsVersion(versionData, defFromFile, *i.asVersion); err != nil { + return fmt.Errorf("Insertion failed: %v", err) + } + + i.printDef(defFromFile) + + } + + if *i.upstreamTarget == "" && *i.orgID == "" { + return fmt.Errorf("No upstream target or org ID defined, these are both required") + } + + // Create the API with the blueprint + bp, err := i.bluePrintLoadFile(*i.input) + if err != nil { + return fmt.Errorf("File load error: %v", err) + } + + def, err := bp.ToAPIDefinition(*i.orgID, *i.upstreamTarget, *i.asMock) + if err != nil { + return fmt.Errorf("Failed to create API Definition from file") + } + + i.printDef(def) + return nil +} + +func (i *Importer) handleSwaggerMode() error { + if *i.createAPI { + if *i.upstreamTarget != "" && *i.orgID != "" { + // Create the API with the blueprint + s, err := i.swaggerLoadFile(*i.input) + if err != nil { + return fmt.Errorf("File load error: %v", err) + } + + def, err := s.ToAPIDefinition(*i.orgID, *i.upstreamTarget, *i.asMock) + if err != nil { + return fmt.Errorf("Failed to create API Defintition from file") + } + + i.printDef(def) + return nil + } + + return fmt.Errorf("No upstream target or org ID defined, these are both required") + + } + + // Different branch, here we need an API Definition to modify + if *i.forAPI == "" { + return fmt.Errorf("If adding to an API, the path to the definition must be listed") + } + + if *i.asVersion == "" { + return fmt.Errorf("No version defined for this import operation, please set an import ID using the --as-version flag") + } + + defFromFile, err := i.apiDefLoadFile(*i.forAPI) + if err != nil { + return fmt.Errorf("failed to load and decode file data for API Definition: %v", err) + } + + s, err := i.swaggerLoadFile(*i.input) + if err != nil { + return fmt.Errorf("File load error: %v", err) + } + + versionData, err := s.ConvertIntoApiVersion(*i.asMock) + if err != nil { + return fmt.Errorf("Conversion into API Def failed: %v", err) + } + + if err := s.InsertIntoAPIDefinitionAsVersion(versionData, defFromFile, *i.asVersion); err != nil { + return fmt.Errorf("Insertion failed: %v", err) + } + + i.printDef(defFromFile) + + return nil +} + +func (i *Importer) printDef(def *apidef.APIDefinition) { + asJSON, err := json.MarshalIndent(def, "", " ") + if err != nil { + log.Error("Marshalling failed: ", err) + } + + // The id attribute is for BSON only and breaks the parser if it's empty, cull it here. + fixed := strings.Replace(string(asJSON), ` "id": "",`, "", 1) + fmt.Println(fixed) +} + +func (i *Importer) swaggerLoadFile(path string) (*importer.SwaggerAST, error) { + swagger, err := importer.GetImporterForSource(importer.SwaggerSource) + if err != nil { + return nil, err + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + if err := swagger.LoadFrom(f); err != nil { + return nil, err + } + + return swagger.(*importer.SwaggerAST), nil +} + +func (i *Importer) bluePrintLoadFile(path string) (*importer.BluePrintAST, error) { + blueprint, err := importer.GetImporterForSource(importer.ApiaryBluePrint) + if err != nil { + return nil, err + } + + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + if err := blueprint.LoadFrom(f); err != nil { + return nil, err + } + + return blueprint.(*importer.BluePrintAST), nil +} + +func (i *Importer) apiDefLoadFile(path string) (*apidef.APIDefinition, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + def := &apidef.APIDefinition{} + if err := json.NewDecoder(f).Decode(&def); err != nil { + return nil, err + } + return def, nil +} diff --git a/lint/lint.go b/cli/lint/lint.go similarity index 100% rename from lint/lint.go rename to cli/lint/lint.go diff --git a/lint/lint_test.go b/cli/lint/lint_test.go similarity index 99% rename from lint/lint_test.go rename to cli/lint/lint_test.go index 753977d9632..fb874dd4a8a 100644 --- a/lint/lint_test.go +++ b/cli/lint/lint_test.go @@ -13,7 +13,7 @@ import ( func TestMain(m *testing.M) { // Use the root package, as that's where the directories and // files required to run the gateway are. - os.Chdir("..") + os.Chdir("../..") os.Exit(m.Run()) } diff --git a/lint/schema.go b/cli/lint/schema.go similarity index 100% rename from lint/schema.go rename to cli/lint/schema.go diff --git a/command_mode.go b/command_mode.go deleted file mode 100644 index f494e3f172e..00000000000 --- a/command_mode.go +++ /dev/null @@ -1,204 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - "strings" - - "github.com/TykTechnologies/tyk/apidef" - "github.com/TykTechnologies/tyk/apidef/importer" -) - -var commandModeOptions = []interface{}{ - importBlueprint, - importSwagger, - createAPI, - orgID, - upstreamTarget, - asMock, - forAPI, - asVersion, -} - -// ./tyk --import-blueprint=blueprint.json --create-api --org-id= --upstream-target="http://widgets.com/api/"` -func handleCommandModeArgs() { - if *importBlueprint != "" { - if err := handleBluePrintMode(); err != nil { - log.Error(err) - } - } - - if *importSwagger != "" { - if err := handleSwaggerMode(); err != nil { - log.Error(err) - } - } -} - -func handleBluePrintMode() error { - if !*createAPI { - // Different branch, here we need an API Definition to modify - if *forAPI == "" { - return fmt.Errorf("If adding to an API, the path to the definition must be listed") - } - - if *asVersion == "" { - return fmt.Errorf("No version defined for this import operation, please set an import ID using the --as-version flag") - } - - defFromFile, err := apiDefLoadFile(*forAPI) - if err != nil { - return fmt.Errorf("failed to load and decode file data for API Definition: %v", err) - } - - bp, err := bluePrintLoadFile(*importBlueprint) - if err != nil { - return fmt.Errorf("File load error: %v", err) - } - versionData, err := bp.ConvertIntoApiVersion(*asMock) - if err != nil { - return fmt.Errorf("onversion into API Def failed: %v", err) - } - - if err := bp.InsertIntoAPIDefinitionAsVersion(versionData, defFromFile, *asVersion); err != nil { - return fmt.Errorf("Insertion failed: %v", err) - } - - printDef(defFromFile) - - } - - if *upstreamTarget == "" && *orgID == "" { - return fmt.Errorf("No upstream target or org ID defined, these are both required") - } - - // Create the API with the blueprint - bp, err := bluePrintLoadFile(*importBlueprint) - if err != nil { - return fmt.Errorf("File load error: %v", err) - } - - def, err := bp.ToAPIDefinition(*orgID, *upstreamTarget, *asMock) - if err != nil { - return fmt.Errorf("Failed to create API Definition from file") - } - - printDef(def) - return nil -} - -func handleSwaggerMode() error { - if *createAPI { - if *upstreamTarget != "" && *orgID != "" { - // Create the API with the blueprint - s, err := swaggerLoadFile(*importSwagger) - if err != nil { - return fmt.Errorf("File load error: %v", err) - } - - def, err := s.ToAPIDefinition(*orgID, *upstreamTarget, *asMock) - if err != nil { - return fmt.Errorf("Failed to create API Defintition from file") - } - - printDef(def) - return nil - } - - return fmt.Errorf("No upstream target or org ID defined, these are both required") - - } - - // Different branch, here we need an API Definition to modify - if *forAPI == "" { - return fmt.Errorf("If adding to an API, the path to the definition must be listed") - } - - if *asVersion == "" { - return fmt.Errorf("No version defined for this import operation, please set an import ID using the --as-version flag") - } - - defFromFile, err := apiDefLoadFile(*forAPI) - if err != nil { - return fmt.Errorf("failed to load and decode file data for API Definition: %v", err) - } - - s, err := swaggerLoadFile(*importSwagger) - if err != nil { - return fmt.Errorf("File load error: %v", err) - } - - versionData, err := s.ConvertIntoApiVersion(*asMock) - if err != nil { - return fmt.Errorf("Conversion into API Def failed: %v", err) - } - - if err := s.InsertIntoAPIDefinitionAsVersion(versionData, defFromFile, *asVersion); err != nil { - return fmt.Errorf("Insertion failed: %v", err) - } - - printDef(defFromFile) - - return nil -} - -func printDef(def *apidef.APIDefinition) { - asJSON, err := json.MarshalIndent(def, "", " ") - if err != nil { - log.Error("Marshalling failed: ", err) - } - - // The id attribute is for BSON only and breaks the parser if it's empty, cull it here. - fixed := strings.Replace(string(asJSON), ` "id": "",`, "", 1) - fmt.Println(fixed) -} - -func swaggerLoadFile(path string) (*importer.SwaggerAST, error) { - swagger, err := importer.GetImporterForSource(importer.SwaggerSource) - if err != nil { - return nil, err - } - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - if err := swagger.LoadFrom(f); err != nil { - return nil, err - } - - return swagger.(*importer.SwaggerAST), nil -} - -func bluePrintLoadFile(path string) (*importer.BluePrintAST, error) { - blueprint, err := importer.GetImporterForSource(importer.ApiaryBluePrint) - if err != nil { - return nil, err - } - - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - if err := blueprint.LoadFrom(f); err != nil { - return nil, err - } - - return blueprint.(*importer.BluePrintAST), nil -} - -func apiDefLoadFile(path string) (*apidef.APIDefinition, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - def := &apidef.APIDefinition{} - if err := json.NewDecoder(f).Decode(&def); err != nil { - return nil, err - } - return def, nil -} diff --git a/gateway_test.go b/gateway_test.go index c41f24289dc..8327c67ba87 100644 --- a/gateway_test.go +++ b/gateway_test.go @@ -24,6 +24,7 @@ import ( "gopkg.in/vmihailenco/msgpack.v2" "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/cli" "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/test" @@ -131,6 +132,8 @@ func TestMain(m *testing.M) { panic(err) } + cli.Init(VERSION, confPaths) + initialiseSystem() // Small part of start() loadAPIEndpoints(mainRouter) @@ -713,8 +716,8 @@ func TestControlListener(t *testing.T) { } func TestHttpPprof(t *testing.T) { - old := httpProfile - defer func() { httpProfile = old }() + old := cli.HTTPProfile + defer func() { cli.HTTPProfile = old }() ts := newTykTestServer(tykTestServerConfig{ sepatateControlAPI: true, @@ -726,7 +729,7 @@ func TestHttpPprof(t *testing.T) { }...) ts.Close() - *httpProfile = true + *cli.HTTPProfile = true ts.Start() ts.Run(t, []test.TestCase{ diff --git a/instrumentation_handlers.go b/instrumentation_handlers.go index c96ba501d97..0c441dccd9f 100644 --- a/instrumentation_handlers.go +++ b/instrumentation_handlers.go @@ -9,6 +9,7 @@ import ( "github.com/gocraft/health" + "github.com/TykTechnologies/tyk/cli" "github.com/TykTechnologies/tyk/request" "github.com/TykTechnologies/tyk/config" @@ -20,7 +21,7 @@ var instrument = health.NewStream() // setupInstrumentation handles all the intialisation of the instrumentation handler func setupInstrumentation() { switch { - case *logInstrumentation: + case *cli.LogInstrumentation: case os.Getenv("TYK_INSTRUMENTATION") == "1": default: return diff --git a/main.go b/main.go index 54a8f14c7fe..d56ca3a60c5 100644 --- a/main.go +++ b/main.go @@ -35,14 +35,13 @@ import ( "github.com/lonelycode/osin" "github.com/rs/cors" "github.com/satori/go.uuid" - "gopkg.in/alecthomas/kingpin.v2" "rsc.io/letsencrypt" "github.com/TykTechnologies/goagain" "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/certs" + cli "github.com/TykTechnologies/tyk/cli" "github.com/TykTechnologies/tyk/config" - "github.com/TykTechnologies/tyk/lint" logger "github.com/TykTechnologies/tyk/log" "github.com/TykTechnologies/tyk/regexp" "github.com/TykTechnologies/tyk/storage" @@ -86,27 +85,6 @@ var ( runningTests = false - version = kingpin.Version(VERSION) - help = kingpin.CommandLine.HelpFlag.Short('h') - conf = kingpin.Flag("conf", "load a named configuration file").PlaceHolder("FILE").String() - port = kingpin.Flag("port", "listen on PORT (overrides config file)").String() - memProfile = kingpin.Flag("memprofile", "generate a memory profile").Bool() - cpuProfile = kingpin.Flag("cpuprofile", "generate a cpu profile").Bool() - blockProfile = kingpin.Flag("blockprofile", "generate a block profile").Bool() - mutexProfile = kingpin.Flag("mutexprofile", "generate a mutex profile").Bool() - httpProfile = kingpin.Flag("httpprofile", "expose runtime profiling data via HTTP").Bool() - debugMode = kingpin.Flag("debug", "enable debug mode").Bool() - importBlueprint = kingpin.Flag("import-blueprint", "import an API Blueprint file").PlaceHolder("FILE").String() - importSwagger = kingpin.Flag("import-swagger", "import a Swagger file").PlaceHolder("FILE").String() - createAPI = kingpin.Flag("create-api", "creates a new API definition from the blueprint").Bool() - orgID = kingpin.Flag("org-id", "assign the API Definition to this org_id (required with create-api").String() - upstreamTarget = kingpin.Flag("upstream-target", "set the upstream target for the definition").PlaceHolder("URL").String() - asMock = kingpin.Flag("as-mock", "creates the API as a mock based on example fields").Bool() - forAPI = kingpin.Flag("for-api", "adds blueprint to existing API Definition as version").PlaceHolder("PATH").String() - asVersion = kingpin.Flag("as-version", "the version number to use when inserting").PlaceHolder("VERSION").String() - logInstrumentation = kingpin.Flag("log-intrumentation", "output intrumentation output to stdout").Bool() - subcmd = kingpin.Arg("subcmd", "run a Tyk subcommand i.e. lint").String() - // confPaths is the series of paths to try to use as config files. The // first one to exist will be used. If none exists, a default config // will be written to the first path in the list. @@ -375,7 +353,7 @@ func loadAPIEndpoints(muxer *mux.Router) { mainLog.Info("Control API hostname set: ", hostname) } - if *httpProfile { + if *cli.HTTPProfile { muxer.HandleFunc("/debug/pprof/profile", pprof_http.Profile) muxer.HandleFunc("/debug/pprof/{_:.*}", pprof_http.Index) } @@ -777,26 +755,6 @@ func setupLogger() { } func initialiseSystem() error { - - // Enable command mode - for _, opt := range commandModeOptions { - switch x := opt.(type) { - case *string: - if *x == "" { - continue - } - case *bool: - if !*x { - continue - } - default: - panic("unexpected type") - } - handleCommandModeArgs() - os.Exit(0) - - } - if runningTests && os.Getenv("TYK_LOGLEVEL") == "" { // `go test` without TYK_LOGLEVEL set defaults to no log // output @@ -804,14 +762,14 @@ func initialiseSystem() error { log.Out = ioutil.Discard gorpc.SetErrorLogger(func(string, ...interface{}) {}) stdlog.SetOutput(ioutil.Discard) - } else if *debugMode { + } else if *cli.DebugMode { log.Level = logrus.DebugLevel mainLog.Debug("Enabling debug-level output") } - if *conf != "" { - mainLog.Debugf("Using %s for configuration", *conf) - confPaths = []string{*conf} + if *cli.Conf != "" { + mainLog.Debugf("Using %s for configuration", *cli.Conf) + confPaths = []string{*cli.Conf} } else { mainLog.Debug("No configuration file defined, will try to use default (tyk.conf)") } @@ -828,7 +786,7 @@ func initialiseSystem() error { config.SetGlobal(globalConf) } - if os.Getenv("TYK_LOGLEVEL") == "" && !*debugMode { + if os.Getenv("TYK_LOGLEVEL") == "" && !*cli.DebugMode { level := strings.ToLower(config.Global().LogLevel) switch level { case "", "info": @@ -850,8 +808,8 @@ func initialiseSystem() error { setupGlobals() - if *port != "" { - portNum, err := strconv.Atoi(*port) + if *cli.Port != "" { + portNum, err := strconv.Atoi(*cli.Port) if err != nil { mainLog.Error("Port specified in flags must be a number: ", err) } else { @@ -926,24 +884,8 @@ func getGlobalStorageHandler(keyPrefix string, hashKeys bool) storage.Handler { } func main() { - kingpin.Parse() - - if *subcmd == "lint" { - path, lines, err := lint.Run(confPaths) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - if len(lines) == 0 { - fmt.Printf("found no issues in %s\n", path) - return - } - fmt.Printf("issues found in %s:\n", path) - for _, line := range lines { - fmt.Println(line) - } - os.Exit(1) - } + cli.Init(VERSION, confPaths) + cli.Parse() NodeID = "solo-" + uuid.NewV4().String() @@ -996,7 +938,7 @@ func main() { } mainLog.Info("Redis connection pools are ready") - if *memProfile { + if *cli.MemProfile { mainLog.Debug("Memory profiling active") var err error if memProfFile, err = os.Create("tyk.mprof"); err != nil { @@ -1004,7 +946,7 @@ func main() { } defer memProfFile.Close() } - if *cpuProfile { + if *cli.CPUProfile { mainLog.Info("Cpu profiling active") cpuProfFile, err := os.Create("tyk.prof") if err != nil { @@ -1013,11 +955,11 @@ func main() { pprof.StartCPUProfile(cpuProfFile) defer pprof.StopCPUProfile() } - if *blockProfile { + if *cli.BlockProfile { mainLog.Info("Block profiling active") runtime.SetBlockProfileRate(1) } - if *mutexProfile { + if *cli.MutexProfile { mainLog.Info("Mutex profiling active") runtime.SetMutexProfileFraction(1) } @@ -1073,7 +1015,7 @@ func main() { } func writeProfiles() { - if *blockProfile { + if *cli.BlockProfile { f, err := os.Create("tyk.blockprof") if err != nil { panic(err) @@ -1083,7 +1025,7 @@ func writeProfiles() { } f.Close() } - if *mutexProfile { + if *cli.MutexProfile { f, err := os.Create("tyk.mutexprof") if err != nil { panic(err)