Skip to content

feat(go): remove murmur hash go deps #3217

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chaokunyang merged 1 commit into from
Jan 27, 2026
Merged

feat(go): remove murmur hash go deps #3217

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion go/fory/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ module github.com/apache/fory/go/fory
go 1.24.0

require (
github.com/spaolacci/murmur3 v1.1.0
github.com/stretchr/testify v1.7.0
golang.org/x/tools v0.39.0
)
Expand Down
2 changes: 0 additions & 2 deletions go/fory/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
Expand Down
5 changes: 2 additions & 3 deletions go/fory/meta_string_resolver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"encoding/binary"
"fmt"
"github.com/apache/fory/go/fory/meta"
"github.com/spaolacci/murmur3"
)

// Constants for string handling
Expand Down Expand Up @@ -221,7 +220,7 @@ func (r *MetaStringResolver) GetMetaStrBytes(metastr *meta.MetaString) *MetaStri
hashcode = ((v1*31 + v2) >> 8 << 8) | int64(metastr.GetEncoding())
} else {
// Large string: use MurmurHash3
h64 := murmur3.Sum64WithSeed(data, 47)
h64 := Murmur3Sum64WithSeed(data, 47)
hashcode = int64((h64 >> 8) << 8)
hashcode |= int64(metastr.GetEncoding())
}
Expand Down Expand Up @@ -249,7 +248,7 @@ func ComputeMetaStringHash(data []byte, encoding meta.Encoding) int64 {
hashcode = ((v1*31 + v2) >> 8 << 8) | int64(encoding)
} else {
// Large string: use MurmurHash3
h64 := murmur3.Sum64WithSeed(data, 47)
h64 := Murmur3Sum64WithSeed(data, 47)
hashcode = int64((h64 >> 8) << 8)
hashcode |= int64(encoding)
}
Expand Down
144 changes: 144 additions & 0 deletions go/fory/murmur.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package fory

import "encoding/binary"

const (
murmurC1_128 = 0x87c37b91114253d5
murmurC2_128 = 0x4cf5ad432745937f
)

func Murmur3Sum64WithSeed(data []byte, seed uint32) uint64 {
h1, _ := Murmur3Sum128WithSeed(data, seed)
return h1
}

func MurmurHash3_x64_128(data []byte, seed int64) (uint64, uint64) {
return Murmur3Sum128WithSeed(data, uint32(seed))
}

func Murmur3Sum128WithSeed(data []byte, seed uint32) (uint64, uint64) {
h1 := uint64(seed)
h2 := uint64(seed)

nblocks := len(data) / 16
for i := 0; i < nblocks; i++ {
block := data[i*16:]
k1 := binary.LittleEndian.Uint64(block)
k2 := binary.LittleEndian.Uint64(block[8:])

k1 *= murmurC1_128
k1 = (k1 << 31) | (k1 >> 33)
k1 *= murmurC2_128
h1 ^= k1

h1 = (h1 << 27) | (h1 >> 37)
h1 += h2
h1 = h1*5 + 0x52dce729

k2 *= murmurC2_128
k2 = (k2 << 33) | (k2 >> 31)
k2 *= murmurC1_128
h2 ^= k2

h2 = (h2 << 31) | (h2 >> 33)
h2 += h1
h2 = h2*5 + 0x38495ab5
}

tail := data[nblocks*16:]
var k1, k2 uint64
switch len(tail) & 15 {
case 15:
k2 ^= uint64(tail[14]) << 48
fallthrough
case 14:
k2 ^= uint64(tail[13]) << 40
fallthrough
case 13:
k2 ^= uint64(tail[12]) << 32
fallthrough
case 12:
k2 ^= uint64(tail[11]) << 24
fallthrough
case 11:
k2 ^= uint64(tail[10]) << 16
fallthrough
case 10:
k2 ^= uint64(tail[9]) << 8
fallthrough
case 9:
k2 ^= uint64(tail[8])
k2 *= murmurC2_128
k2 = (k2 << 33) | (k2 >> 31)
k2 *= murmurC1_128
h2 ^= k2
fallthrough
case 8:
k1 ^= uint64(tail[7]) << 56
fallthrough
case 7:
k1 ^= uint64(tail[6]) << 48
fallthrough
case 6:
k1 ^= uint64(tail[5]) << 40
fallthrough
case 5:
k1 ^= uint64(tail[4]) << 32
fallthrough
case 4:
k1 ^= uint64(tail[3]) << 24
fallthrough
case 3:
k1 ^= uint64(tail[2]) << 16
fallthrough
case 2:
k1 ^= uint64(tail[1]) << 8
fallthrough
case 1:
k1 ^= uint64(tail[0])
k1 *= murmurC1_128
k1 = (k1 << 31) | (k1 >> 33)
k1 *= murmurC2_128
h1 ^= k1
}

h1 ^= uint64(len(data))
h2 ^= uint64(len(data))

h1 += h2
h2 += h1

h1 = murmurFmix64(h1)
h2 = murmurFmix64(h2)

h1 += h2
h2 += h1

return h1, h2
}

func murmurFmix64(k uint64) uint64 {
k ^= k >> 33
k *= 0xff51afd7ed558ccd
k ^= k >> 33
k *= 0xc4ceb9fe1a85ec53
k ^= k >> 33
return k
}
4 changes: 1 addition & 3 deletions go/fory/struct.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@ import (
"unicode"
"unicode/utf8"
"unsafe"

"github.com/spaolacci/murmur3"
)

// GetStructHash returns the struct hash for a given type using the provided TypeResolver.
Expand Down Expand Up @@ -970,7 +968,7 @@ func (s *structSerializer) computeHash() int32 {

hashString := ComputeStructFingerprint(fields)
data := []byte(hashString)
h1, _ := murmur3.Sum128WithSeed(data, 47)
h1, _ := Murmur3Sum128WithSeed(data, 47)
hash := int32(h1 & 0xFFFFFFFF)

if DebugOutputEnabled() {
Expand Down
14 changes: 3 additions & 11 deletions go/fory/tests/xlang/xlang_test_main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
"runtime"

"github.com/apache/fory/go/fory"
"github.com/spaolacci/murmur3"
)

// ============================================================================
Expand Down Expand Up @@ -183,13 +182,6 @@ func assertEqualFloat64(expected, actual float64, name string) {
}
}

func murmurHash3_x64_128(data []byte, seed int64) (uint64, uint64) {
h := murmur3.New128WithSeed(uint32(seed))
h.Write(data)
h1, h2 := h.Sum128()
return h1, h2
}

// ============================================================================
// Test Data Structures
// ============================================================================
Expand Down Expand Up @@ -559,8 +551,8 @@ func testMurmurHash3() {
_ = buf.ReadInt64(&bufErr)
_ = buf.ReadInt64(&bufErr)

h1_1, h1_2 := murmurHash3_x64_128([]byte{1, 2, 8}, 47)
h2_1, h2_2 := murmurHash3_x64_128([]byte("01234567890123456789"), 47)
h1_1, h1_2 := fory.MurmurHash3_x64_128([]byte{1, 2, 8}, 47)
h2_1, h2_2 := fory.MurmurHash3_x64_128([]byte("01234567890123456789"), 47)

outBuf := fory.NewByteBuffer(make([]byte, 0, 32))
outBuf.WriteInt64(int64(h1_1))
Expand All @@ -575,7 +567,7 @@ func testMurmurHash3() {
h2 := buf.ReadInt64(&bufErr)

// Compute expected values
expected1, expected2 := murmurHash3_x64_128([]byte{1, 2, 8}, 47)
expected1, expected2 := fory.MurmurHash3_x64_128([]byte{1, 2, 8}, 47)

if h1 != int64(expected1) || h2 != int64(expected2) {
panic(fmt.Sprintf("MurmurHash3 mismatch: got (%d, %d), expected (%d, %d)",
Expand Down
3 changes: 1 addition & 2 deletions go/fory/type_def.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"reflect"

"github.com/apache/fory/go/fory/meta"
"github.com/spaolacci/murmur3"
)

const (
Expand Down Expand Up @@ -1233,7 +1232,7 @@ func prependGlobalHeader(buffer *ByteBuffer, isCompressed bool, hasFieldsMeta bo
var header uint64
metaSize := buffer.WriterIndex()

hashValue := murmur3.Sum64WithSeed(buffer.GetByteSlice(0, metaSize), 47)
hashValue := Murmur3Sum64WithSeed(buffer.GetByteSlice(0, metaSize), 47)
header |= hashValue << (64 - NUM_HASH_BITS)

if hasFieldsMeta {
Expand Down
Loading