Permalink
Browse files

mime: use sync.Map instead of RWMutex for type lookups

This provides a significant speedup for TypeByExtension and
ExtensionsByType when using many CPU cores.

updates #17973
updates #18177

name                                          old time/op    new time/op    delta
QEncodeWord                                      526ns ± 3%     525ns ± 3%     ~     (p=0.990 n=15+28)
QEncodeWord-6                                    945ns ± 7%     913ns ±20%     ~     (p=0.220 n=14+28)
QEncodeWord-48                                  1.02µs ± 2%    1.00µs ± 6%   -2.22%  (p=0.036 n=13+27)
QDecodeWord                                      311ns ±18%     323ns ±20%     ~     (p=0.107 n=16+28)
QDecodeWord-6                                    595ns ±12%     612ns ±11%     ~     (p=0.093 n=15+27)
QDecodeWord-48                                   592ns ± 6%     606ns ± 8%   +2.39%  (p=0.045 n=16+26)
QDecodeHeader                                    389ns ± 4%     394ns ± 8%     ~     (p=0.161 n=12+26)
QDecodeHeader-6                                  685ns ±12%     674ns ±20%     ~     (p=0.773 n=14+27)
QDecodeHeader-48                                 658ns ±13%     669ns ±14%     ~     (p=0.457 n=16+28)
TypeByExtension/.html                           77.4ns ±15%    55.5ns ±13%  -28.35%  (p=0.000 n=8+8)
TypeByExtension/.html-6                          263ns ± 9%      10ns ±21%  -96.29%  (p=0.000 n=8+8)
TypeByExtension/.html-48                         175ns ± 5%       2ns ±16%  -98.88%  (p=0.000 n=8+8)
TypeByExtension/.HTML                            113ns ± 6%      97ns ± 6%  -14.37%  (p=0.000 n=8+8)
TypeByExtension/.HTML-6                          273ns ± 7%      17ns ± 4%  -93.93%  (p=0.000 n=7+8)
TypeByExtension/.HTML-48                         175ns ± 4%       4ns ± 4%  -97.73%  (p=0.000 n=8+8)
TypeByExtension/.unused                          116ns ± 4%      90ns ± 4%  -22.89%  (p=0.001 n=7+7)
TypeByExtension/.unused-6                        262ns ± 5%      15ns ± 4%  -94.17%  (p=0.000 n=8+8)
TypeByExtension/.unused-48                       176ns ± 4%       3ns ±10%  -98.10%  (p=0.000 n=8+8)
ExtensionsByType/text/html                       630ns ± 5%     522ns ± 5%  -17.19%  (p=0.000 n=8+7)
ExtensionsByType/text/html-6                     314ns ±20%     136ns ± 6%  -56.80%  (p=0.000 n=8+8)
ExtensionsByType/text/html-48                    298ns ± 4%     104ns ± 6%  -65.06%  (p=0.000 n=8+8)
ExtensionsByType/text/html;_charset=utf-8       1.12µs ± 3%    1.05µs ± 7%   -6.19%  (p=0.004 n=8+7)
ExtensionsByType/text/html;_charset=utf-8-6      402ns ±11%     307ns ± 4%  -23.77%  (p=0.000 n=8+8)
ExtensionsByType/text/html;_charset=utf-8-48     422ns ± 3%     309ns ± 4%  -26.86%  (p=0.000 n=8+8)
ExtensionsByType/application/octet-stream        810ns ± 2%     747ns ± 5%   -7.74%  (p=0.000 n=8+8)
ExtensionsByType/application/octet-stream-6      289ns ± 9%     185ns ± 8%  -36.15%  (p=0.000 n=7+8)
ExtensionsByType/application/octet-stream-48     267ns ± 6%      94ns ± 2%  -64.91%  (p=0.000 n=8+7)

name                                          old alloc/op   new alloc/op   delta
QEncodeWord                                      48.0B ± 0%     48.0B ± 0%     ~     (all equal)
QEncodeWord-6                                    48.0B ± 0%     48.0B ± 0%     ~     (all equal)
QEncodeWord-48                                   48.0B ± 0%     48.0B ± 0%     ~     (all equal)
QDecodeWord                                      48.0B ± 0%     48.0B ± 0%     ~     (all equal)
QDecodeWord-6                                    48.0B ± 0%     48.0B ± 0%     ~     (all equal)
QDecodeWord-48                                   48.0B ± 0%     48.0B ± 0%     ~     (all equal)
QDecodeHeader                                    48.0B ± 0%     48.0B ± 0%     ~     (all equal)
QDecodeHeader-6                                  48.0B ± 0%     48.0B ± 0%     ~     (all equal)
QDecodeHeader-48                                 48.0B ± 0%     48.0B ± 0%     ~     (all equal)
TypeByExtension/.html                            0.00B          0.00B          ~     (all equal)
TypeByExtension/.html-6                          0.00B          0.00B          ~     (all equal)
TypeByExtension/.html-48                         0.00B          0.00B          ~     (all equal)
TypeByExtension/.HTML                            0.00B          0.00B          ~     (all equal)
TypeByExtension/.HTML-6                          0.00B          0.00B          ~     (all equal)
TypeByExtension/.HTML-48                         0.00B          0.00B          ~     (all equal)
TypeByExtension/.unused                          0.00B          0.00B          ~     (all equal)
TypeByExtension/.unused-6                        0.00B          0.00B          ~     (all equal)
TypeByExtension/.unused-48                       0.00B          0.00B          ~     (all equal)
ExtensionsByType/text/html                        192B ± 0%      176B ± 0%   -8.33%  (p=0.000 n=8+8)
ExtensionsByType/text/html-6                      192B ± 0%      176B ± 0%   -8.33%  (p=0.000 n=8+8)
ExtensionsByType/text/html-48                     192B ± 0%      176B ± 0%   -8.33%  (p=0.000 n=8+8)
ExtensionsByType/text/html;_charset=utf-8         480B ± 0%      464B ± 0%   -3.33%  (p=0.000 n=8+8)
ExtensionsByType/text/html;_charset=utf-8-6       480B ± 0%      464B ± 0%   -3.33%  (p=0.000 n=8+8)
ExtensionsByType/text/html;_charset=utf-8-48      480B ± 0%      464B ± 0%   -3.33%  (p=0.000 n=8+8)
ExtensionsByType/application/octet-stream         160B ± 0%      160B ± 0%     ~     (all equal)
ExtensionsByType/application/octet-stream-6       160B ± 0%      160B ± 0%     ~     (all equal)
ExtensionsByType/application/octet-stream-48      160B ± 0%      160B ± 0%     ~     (all equal)

name                                          old allocs/op  new allocs/op  delta
QEncodeWord                                       1.00 ± 0%      1.00 ± 0%     ~     (all equal)
QEncodeWord-6                                     1.00 ± 0%      1.00 ± 0%     ~     (all equal)
QEncodeWord-48                                    1.00 ± 0%      1.00 ± 0%     ~     (all equal)
QDecodeWord                                       2.00 ± 0%      2.00 ± 0%     ~     (all equal)
QDecodeWord-6                                     2.00 ± 0%      2.00 ± 0%     ~     (all equal)
QDecodeWord-48                                    2.00 ± 0%      2.00 ± 0%     ~     (all equal)
QDecodeHeader                                     2.00 ± 0%      2.00 ± 0%     ~     (all equal)
QDecodeHeader-6                                   2.00 ± 0%      2.00 ± 0%     ~     (all equal)
QDecodeHeader-48                                  2.00 ± 0%      2.00 ± 0%     ~     (all equal)
TypeByExtension/.html                             0.00           0.00          ~     (all equal)
TypeByExtension/.html-6                           0.00           0.00          ~     (all equal)
TypeByExtension/.html-48                          0.00           0.00          ~     (all equal)
TypeByExtension/.HTML                             0.00           0.00          ~     (all equal)
TypeByExtension/.HTML-6                           0.00           0.00          ~     (all equal)
TypeByExtension/.HTML-48                          0.00           0.00          ~     (all equal)
TypeByExtension/.unused                           0.00           0.00          ~     (all equal)
TypeByExtension/.unused-6                         0.00           0.00          ~     (all equal)
TypeByExtension/.unused-48                        0.00           0.00          ~     (all equal)
ExtensionsByType/text/html                        3.00 ± 0%      3.00 ± 0%     ~     (all equal)
ExtensionsByType/text/html-6                      3.00 ± 0%      3.00 ± 0%     ~     (all equal)
ExtensionsByType/text/html-48                     3.00 ± 0%      3.00 ± 0%     ~     (all equal)
ExtensionsByType/text/html;_charset=utf-8         4.00 ± 0%      4.00 ± 0%     ~     (all equal)
ExtensionsByType/text/html;_charset=utf-8-6       4.00 ± 0%      4.00 ± 0%     ~     (all equal)
ExtensionsByType/text/html;_charset=utf-8-48      4.00 ± 0%      4.00 ± 0%     ~     (all equal)
ExtensionsByType/application/octet-stream         2.00 ± 0%      2.00 ± 0%     ~     (all equal)
ExtensionsByType/application/octet-stream-6       2.00 ± 0%      2.00 ± 0%     ~     (all equal)
ExtensionsByType/application/octet-stream-48      2.00 ± 0%      2.00 ± 0%     ~     (all equal)

https://perf.golang.org/search?q=upload:20170427.4

Change-Id: I35438be087ad6eb3d5da9119b395723ea5babaf6
Reviewed-on: https://go-review.googlesource.com/41990
Run-TryBot: Bryan Mills <bcmills@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>
  • Loading branch information...
bcmills committed Feb 16, 2017
1 parent ce5263f commit e8d7e5d1fa7d8477b91cb4dffeac57c7c20cb5c5
Showing with 56 additions and 52 deletions.
  1. +56 −52 src/mime/type.go
View
@@ -12,24 +12,48 @@ import (
)
var (
mimeLock sync.RWMutex // guards following 3 maps
mimeTypes map[string]string // ".Z" => "application/x-compress"
mimeTypesLower map[string]string // ".z" => "application/x-compress"
mimeTypes sync.Map // map[string]string; ".Z" => "application/x-compress"
mimeTypesLower sync.Map // map[string]string; ".z" => "application/x-compress"
// extensions maps from MIME type to list of lowercase file
// extensions: "image/jpeg" => [".jpg", ".jpeg"]
extensions map[string][]string
extensionsMu sync.Mutex // Guards stores (but not loads) on extensions.
extensions sync.Map // map[string][]string; slice values are append-only.
)
func clearSyncMap(m *sync.Map) {
m.Range(func(k, _ interface{}) bool {
m.Delete(k)
return true
})
}
// setMimeTypes is used by initMime's non-test path, and by tests.
// The two maps must not be the same, or nil.
func setMimeTypes(lowerExt, mixExt map[string]string) {
if lowerExt == nil || mixExt == nil {
panic("nil map")
clearSyncMap(&mimeTypes)
clearSyncMap(&mimeTypesLower)
clearSyncMap(&extensions)
for k, v := range lowerExt {
mimeTypesLower.Store(k, v)
}
for k, v := range mixExt {
mimeTypes.Store(k, v)
}
extensionsMu.Lock()
defer extensionsMu.Unlock()
for k, v := range lowerExt {
justType, _, err := ParseMediaType(v)
if err != nil {
panic(err)
}
var exts []string
if ei, ok := extensions.Load(k); ok {
exts = ei.([]string)
}
extensions.Store(justType, append(exts, k))
}
mimeTypesLower = lowerExt
mimeTypes = mixExt
extensions = invert(lowerExt)
}
var builtinTypesLower = map[string]string{
@@ -45,29 +69,6 @@ var builtinTypesLower = map[string]string{
".xml": "text/xml; charset=utf-8",
}
func clone(m map[string]string) map[string]string {
m2 := make(map[string]string, len(m))
for k, v := range m {
m2[k] = v
if strings.ToLower(k) != k {
panic("keys in builtinTypesLower must be lowercase")
}
}
return m2
}
func invert(m map[string]string) map[string][]string {
m2 := make(map[string][]string, len(m))
for k, v := range m {
justType, _, err := ParseMediaType(v)
if err != nil {
panic(err)
}
m2[justType] = append(m2[justType], k)
}
return m2
}
var once sync.Once // guards initMime
var testInitMime, osInitMime func()
@@ -76,7 +77,7 @@ func initMime() {
if fn := testInitMime; fn != nil {
fn()
} else {
setMimeTypes(builtinTypesLower, clone(builtinTypesLower))
setMimeTypes(builtinTypesLower, builtinTypesLower)
osInitMime()
}
}
@@ -100,12 +101,10 @@ func initMime() {
// Text types have the charset parameter set to "utf-8" by default.
func TypeByExtension(ext string) string {
once.Do(initMime)
mimeLock.RLock()
defer mimeLock.RUnlock()
// Case-sensitive lookup.
if v := mimeTypes[ext]; v != "" {
return v
if v, ok := mimeTypes.Load(ext); ok {
return v.(string)
}
// Case-insensitive lookup.
@@ -118,17 +117,19 @@ func TypeByExtension(ext string) string {
c := ext[i]
if c >= utf8RuneSelf {
// Slow path.
return mimeTypesLower[strings.ToLower(ext)]
si, _ := mimeTypesLower.Load(strings.ToLower(ext))
s, _ := si.(string)
return s
}
if 'A' <= c && c <= 'Z' {
lower = append(lower, c+('a'-'A'))
} else {
lower = append(lower, c)
}
}
// The conversion from []byte to string doesn't allocate in
// a map lookup.
return mimeTypesLower[string(lower)]
si, _ := mimeTypesLower.Load(string(lower))
s, _ := si.(string)
return s
}
// ExtensionsByType returns the extensions known to be associated with the MIME
@@ -142,13 +143,11 @@ func ExtensionsByType(typ string) ([]string, error) {
}
once.Do(initMime)
mimeLock.RLock()
defer mimeLock.RUnlock()
s, ok := extensions[justType]
s, ok := extensions.Load(justType)
if !ok {
return nil, nil
}
return append([]string{}, s...), nil
return append([]string{}, s.([]string)...), nil
}
// AddExtensionType sets the MIME type associated with
@@ -173,15 +172,20 @@ func setExtensionType(extension, mimeType string) error {
}
extLower := strings.ToLower(extension)
mimeLock.Lock()
defer mimeLock.Unlock()
mimeTypes[extension] = mimeType
mimeTypesLower[extLower] = mimeType
for _, v := range extensions[justType] {
mimeTypes.Store(extension, mimeType)
mimeTypesLower.Store(extLower, mimeType)
extensionsMu.Lock()
defer extensionsMu.Unlock()
var exts []string
if ei, ok := extensions.Load(justType); ok {
exts = ei.([]string)
}
for _, v := range exts {
if v == extLower {
return nil
}
}
extensions[justType] = append(extensions[justType], extLower)
extensions.Store(justType, append(exts, extLower))
return nil
}

0 comments on commit e8d7e5d

Please sign in to comment.