Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Concurrent reading from cell is not supported? #861

Closed
listen-lavender opened this issue Jun 16, 2021 · 7 comments
Closed

Concurrent reading from cell is not supported? #861

listen-lavender opened this issue Jun 16, 2021 · 7 comments
Labels
enhancement New feature or request

Comments

@listen-lavender
Copy link

listen-lavender commented Jun 16, 2021

Description

Steps to reproduce the issue:

  1. an Excel file with many picture cell
  2. need concurrent reading from cell
  3. check by go run -race

Test Excel
test_import_goods.xlsx

Test Code

package main

import (
  "strconv"
  "sync"

  "github.com/360EntSecGroup-Skylar/excelize/v2"
)

var tableIndex = map[int]string{
  1: "A", 2: "B", 3: "C", 4: "D", 5: "E",
  6: "F", 7: "G", 8: "H", 9: "I", 10: "J",
  11: "K", 12: "L", 13: "M", 14: "N", 15: "O",
  16: "P", 17: "Q", 18: "R", 19: "S", 20: "T",
  21: "U", 22: "V", 23: "W", 24: "X", 25: "Y", 26: "Z",
}

// 检查导入文件
func main() {
  localFile := "test_import_goods.xlsx"
  f, err := excelize.OpenFile(localFile)
  if err != nil {
    println("UploadGoods", "excelize open error", err.Error())
    return
  }

  var (
    sheet string
  )

  sheets := f.GetSheetList()
  if len(sheets) > 0 {
    sheet = sheets[0]
  } else {
    println("UploadGoods", "no sheet")
    return
  }

  var wg sync.WaitGroup

  sheetRows := getSheetRows(f, sheet) //行数量
  sheetCols := getSheetCols(f, sheet) //列数量

  // time.Sleep(time.Second * 3600)
  for i := 0; i < sheetRows; i++ {
    if i < 2 {
      continue
    }
    for j := 0; j < sheetCols; j++ {
      if j < 9 {
        continue
      }
      wg.Add(1)
      go func(i int, j int) {
        defer func() {
          if r := recover(); r != nil {
            println("ImportCheckImpl error")
          }
        }()
        defer wg.Done()
        if _, _, err := f.GetPicture(sheet, tableIndex[j+1]+strconv.Itoa(i+1)); err != nil {
          println("GetPicture", err.Error())
        }
      }(i, j)
    }
  }
  wg.Wait()
}

//读取行数
func getSheetRows(f *excelize.File, sheet string) int {
  i := 0
  rows, err := f.Rows(sheet)
  if err != nil {
    println("getSheetCols f.Rows", "excelize open error", err.Error())
    return i
  }
  for rows.Next() {
    //读取每行的数据
    cols, err := rows.Columns()
    if err != nil {
      println("getSheetCols rows.Columns()", "row", i, "get columns error", err.Error())
      continue
    }
    cData := ""
    for _, c := range cols {
      cData = cData + c
    }
    if cData == "" { //判断下一行是否为空数据(存在表格显示没有数据但是能读取出空数据的情况)
      break
    }
    i = i + 1
    continue
  }
  return i
}

//读取列数
func getSheetCols(f *excelize.File, sheet string) int {
  i := 0
  cols, err := f.Cols(sheet)
  if err != nil {
    println("getSheetCols f.Rows", "excelize open error", err.Error())
    return i
  }
  for cols.Next() {
    //读取每行的数据
    // cols, err := rcols.Columns()
    cols, err := cols.Rows()
    if err != nil {
      println("getSheetCols rows.Columns()", "row", i, "get columns error", err.Error())
      continue
    }
    cData := ""
    for _, c := range cols {
      cData = cData + c
    }
    if cData == "" { //判断下一行是否为空数据(存在表格显示没有数据但是能读取出空数据的情况)
      break
    }
    i = i + 1
    continue
  }
  return i
}

Describe the results you received:

Describe the results you expected:

Output of go version:

go version go1.14.4 darwin/amd64

Excelize version or commit ID:

excelize/v2@v2.3.1

Output of Error message:

==================
WARNING: DATA RACE
Write at 0x00c000519b90 by goroutine 28:
  runtime.mapassign_faststr()
      /usr/local/Cellar/go/1.14/src/runtime/map_faststr.go:202 +0x0
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).drawingParser()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/drawing.go:1165 +0xcf2
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).getPicture()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/picture.go:513 +0x6d
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).GetPicture()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/picture.go:478 +0x35b
  main.main.func1()
      /Users/lavenderuni/workspace/yunzujia/go/src/code.clouderwork.com/clouderwork/ycd_goods/test/test_import_goods.go:154 +0x20c

Previous read at 0x00c000519b90 by goroutine 29:
  runtime.mapaccess1_faststr()
      /usr/local/Cellar/go/1.14/src/runtime/map_faststr.go:12 +0x0
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).drawingParser()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/drawing.go:1141 +0x9c
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).getPicture()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/picture.go:513 +0x6d
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).GetPicture()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/picture.go:478 +0x35b
  main.main.func1()
      /Users/lavenderuni/workspace/yunzujia/go/src/code.clouderwork.com/clouderwork/ycd_goods/test/test_import_goods.go:154 +0x20c

Goroutine 28 (running) created at:
  main.main()
      /Users/lavenderuni/workspace/yunzujia/go/src/code.clouderwork.com/clouderwork/ycd_goods/test/test_import_goods.go:144 +0xa51

Goroutine 29 (running) created at:
  main.main()
      /Users/lavenderuni/workspace/yunzujia/go/src/code.clouderwork.com/clouderwork/ycd_goods/test/test_import_goods.go:144 +0xa51
==================
==================
WARNING: DATA RACE
Read at 0x00c000a10340 by goroutine 31:
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).getSheetRelationshipsTargetByID()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/picture.go:429 +0x20b
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).GetPicture()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/picture.go:469 +0x191
  main.main.func1()
      /Users/lavenderuni/workspace/yunzujia/go/src/code.clouderwork.com/clouderwork/ycd_goods/test/test_import_goods.go:154 +0x20c

Previous write at 0x00c000a10340 by goroutine 28:
  reflect.typedmemmove()
      /usr/local/Cellar/go/1.14/src/runtime/mbarrier.go:177 +0x0
  reflect.Value.Set()
      /usr/local/Cellar/go/1.14/src/reflect/value.go:1534 +0xfb
  reflect.Append()
      /usr/local/Cellar/go/1.14/src/reflect/value.go:2028 +0x151
  encoding/xml.(*Decoder).unmarshal()
      /usr/local/Cellar/go/1.14/src/encoding/xml/read.go:398 +0x3552
  encoding/xml.(*Decoder).unmarshalPath()
      /usr/local/Cellar/go/1.14/src/encoding/xml/read.go:690 +0x90c
  encoding/xml.(*Decoder).unmarshal()
      /usr/local/Cellar/go/1.14/src/encoding/xml/read.go:524 +0x16a7
  encoding/xml.(*Decoder).DecodeElement()
      /usr/local/Cellar/go/1.14/src/encoding/xml/read.go:151 +0x11e
  encoding/xml.(*Decoder).Decode()
      /usr/local/Cellar/go/1.14/src/encoding/xml/read.go:139 +0x438
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).relsReader()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/sheet.go:1617 +0x44d
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).getSheetRelationshipsTargetByID()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/picture.go:425 +0x19e
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).GetPicture()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/picture.go:469 +0x191
  main.main.func1()
      /Users/lavenderuni/workspace/yunzujia/go/src/code.clouderwork.com/clouderwork/ycd_goods/test/test_import_goods.go:154 +0x20c

Goroutine 31 (running) created at:
  main.main()
      /Users/lavenderuni/workspace/yunzujia/go/src/code.clouderwork.com/clouderwork/ycd_goods/test/test_import_goods.go:144 +0xa51

Goroutine 28 (running) created at:
  main.main()
      /Users/lavenderuni/workspace/yunzujia/go/src/code.clouderwork.com/clouderwork/ycd_goods/test/test_import_goods.go:144 +0xa51
==================
==================
WARNING: DATA RACE
Read at 0x00c00071e178 by goroutine 89:
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).drawingParser()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/drawing.go:1168 +0xdae
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).getPicture()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/picture.go:513 +0x6d
  github.com/360EntSecGroup-Skylar/excelize/v2.(*File).GetPicture()
      /usr/local/Cellar/go/1.14/selfcode/pkg/mod/github.com/360!ent!sec!group-!skylar/excelize/v2@v2.3.1/picture.go:478 +0x35b
  main.main.func1()
      /Users/lavenderuni/workspace/yunzujia/go/src/code.clouderwork.com/clouderwork/ycd_goods/test/test_import_goods.go:154 +0x20c

Previous write at 0x00c00071e178 by goroutine 33:
  [failed to restore the stack]

Goroutine 89 (running) created at:
  main.main()
      /Users/lavenderuni/workspace/yunzujia/go/src/code.clouderwork.com/clouderwork/ycd_goods/test/test_import_goods.go:144 +0xa51

Goroutine 33 (running) created at:
  main.main()
      /Users/lavenderuni/workspace/yunzujia/go/src/code.clouderwork.com/clouderwork/ycd_goods/test/test_import_goods.go:144 +0xa51
==================

Environment details (OS, Microsoft Excel™ version, physical, etc.):

@xuri
Copy link
Member

xuri commented Jun 16, 2021

Thanks for your issue, currently, this library doesn't support reading data concurrently.

@listen-lavender
Copy link
Author

listen-lavender commented Jun 17, 2021

@xuri is it possible to be supported? is it be supported on the way?

@listen-lavender
Copy link
Author

@xuri I want to fork a private repository for the concurrent reading feature. And if i can ignore writing scenario, is it easy to realize my goal? can you help me with any good prompt?

@xuri
Copy link
Member

xuri commented Jun 17, 2021

Use sync.Map instead of the map in the File struct and make all struct reader concurrency safe.

@xuri xuri added the enhancement New feature or request label Jun 17, 2021
@listen-lavender
Copy link
Author

@xuri many thanks

@listen-lavender
Copy link
Author

listen-lavender commented Jun 17, 2021

@xuri Okay, my concurrent reading problem is solved by this issue. I wonder if it is easy to support concurrent writing. If use sync.Map instead of the unsafe map, then concurrent writing can be realized. I would like to contribute this project with a pull request in my spare time.

And this is my temporary solution for concurrent reading.

package excelize

import "sync"

const (
	real_delete = false
)

type RWMapRelationships struct {
	relationships map[string]*xlsxRelationships
	rwl           sync.RWMutex
}

func NewRWMapRelationships() *RWMapRelationships {
	return &RWMapRelationships{
		relationships: make(map[string]*xlsxRelationships),
	}
}

func (rwmap *RWMapRelationships) GetSet(key string, extract func(string) *xlsxRelationships) *xlsxRelationships {
	rwmap.rwl.RLock()
	val := rwmap.relationships[key]
	rwmap.rwl.RUnlock()
	if val != nil {
		return val
	}
	rwmap.rwl.Lock()
	val = rwmap.relationships[key]
	if val == nil {
		if extract != nil {
			val = extract(key)
			rwmap.relationships[key] = val
		}
	}
	rwmap.rwl.Unlock()
	return val
}

func (rwmap *RWMapRelationships) Get(key string) *xlsxRelationships {
	rwmap.rwl.RLock()
	val := rwmap.relationships[key]
	rwmap.rwl.RUnlock()
	return val
}

func (rwmap *RWMapRelationships) Set(key string, val *xlsxRelationships) {
	rwmap.rwl.Lock()
	rwmap.relationships[key] = val
	rwmap.rwl.Unlock()
}

func (rwmap *RWMapRelationships) Delete(key string) {
	rwmap.rwl.Lock()
	if !real_delete {
		rwmap.relationships[key] = nil
	} else {
		delete(rwmap.relationships, key)
	}
	rwmap.rwl.Unlock()
}

func (rwmap *RWMapRelationships) Shadow() map[string]*xlsxRelationships {
	rwmap.rwl.RLock()
	s := make(map[string]*xlsxRelationships, len(rwmap.relationships))
	for key, val := range rwmap.relationships {
		s[key] = val
	}
	rwmap.rwl.RUnlock()
	return s
}

type RWMapDrawings struct {
	drawings map[string]*xlsxWsDr
	rwl      sync.RWMutex
}

func NewRWMapDrawings() *RWMapDrawings {
	return &RWMapDrawings{
		drawings: make(map[string]*xlsxWsDr),
	}
}

func (rwmap *RWMapDrawings) GetSet(key string, extract func(string) *xlsxWsDr) *xlsxWsDr {
	rwmap.rwl.RLock()
	val := rwmap.drawings[key]
	rwmap.rwl.RUnlock()
	if val != nil {
		return val
	}
	rwmap.rwl.Lock()
	val = rwmap.drawings[key]
	if val == nil {
		if extract != nil {
			val = extract(key)
			rwmap.drawings[key] = val
		}
	}
	rwmap.rwl.Unlock()
	return val
}

func (rwmap *RWMapDrawings) Get(key string) *xlsxWsDr {
	rwmap.rwl.RLock()
	val := rwmap.drawings[key]
	rwmap.rwl.RUnlock()
	return val
}

func (rwmap *RWMapDrawings) Set(key string, val *xlsxWsDr) {
	rwmap.rwl.Lock()
	rwmap.drawings[key] = val
	rwmap.rwl.Unlock()
}

func (rwmap *RWMapDrawings) Delete(key string) {
	rwmap.rwl.Lock()
	if !real_delete {
		rwmap.drawings[key] = nil
	} else {
		delete(rwmap.drawings, key)
	}
	rwmap.rwl.Unlock()
}

func (rwmap *RWMapDrawings) Shadow() map[string]*xlsxWsDr {
	rwmap.rwl.RLock()
	s := make(map[string]*xlsxWsDr, len(rwmap.drawings))
	for key, val := range rwmap.drawings {
		s[key] = val
	}
	rwmap.rwl.RUnlock()
	return s
}

@xuri xuri closed this as completed in 0e02329 Jul 4, 2021
@xuri
Copy link
Member

xuri commented Jul 4, 2021

I have added concurrency get cell picture support, please try to upgrade the master branch code, and this feature will be released in the next version.

jenbonzhang pushed a commit to jenbonzhang/excelize that referenced this issue Oct 22, 2023
…ve unused internal function `getSheetNameByID`
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
enhancement New feature or request
Projects
None yet
Development

No branches or pull requests

2 participants