Skip to content

Commit

Permalink
Updates import to read with a scanner (#4788)
Browse files Browse the repository at this point in the history
* Updates import to read with a scanner

* Fix linter
  • Loading branch information
mgdelacroix committed Jul 28, 2023
1 parent be2f587 commit cfa38b8
Showing 1 changed file with 16 additions and 19 deletions.
35 changes: 16 additions & 19 deletions server/app/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@ import (
)

const (
archiveVersion = 2
legacyFileBegin = "{\"version\":1"
archiveVersion = 2
legacyFileBegin = "{\"version\":1"
importMaxFileSize = 1024 * 1024 * 70
)

var (
errBlockIsNotABoard = errors.New("block is not a board")
errBlockIsNotABoard = errors.New("block is not a board")
errSizeLimitExceeded = errors.New("size limit exceeded")
)

// ImportArchive imports an archive containing zero or more boards, plus all
Expand Down Expand Up @@ -129,7 +131,8 @@ func (a *App) ImportBoardJSONL(r io.Reader, opt model.ImportArchiveOptions) (str
Blocks: make([]*model.Block, 0, 10),
Boards: make([]*model.Board, 0, 10),
}
lineReader := bufio.NewReader(r)
lineReader := &io.LimitedReader{R: r, N: importMaxFileSize + 1}
scanner := bufio.NewScanner(lineReader)

userID := opt.ModifiedBy
if userID == model.SingleUser {
Expand All @@ -141,8 +144,12 @@ func (a *App) ImportBoardJSONL(r io.Reader, opt model.ImportArchiveOptions) (str

lineNum := 1
firstLine := true
for {
line, errRead := readLine(lineReader)
for scanner.Scan() {
if lineReader.N <= 0 {
return "", fmt.Errorf("error parsing archive line %d: %w", lineNum, errSizeLimitExceeded)
}

line := bytes.TrimSpace(scanner.Bytes())
if len(line) != 0 {
var skip bool
if firstLine {
Expand Down Expand Up @@ -209,14 +216,10 @@ func (a *App) ImportBoardJSONL(r io.Reader, opt model.ImportArchiveOptions) (str
firstLine = false
}
}
}

if errRead != nil {
if errors.Is(errRead, io.EOF) {
break
}
return "", fmt.Errorf("error reading archive line %d: %w", lineNum, errRead)
}
lineNum++
if errRead := scanner.Err(); errRead != nil {
return "", fmt.Errorf("error reading archive line %d: %w", lineNum, errRead)
}

// loop to remove the people how are not part of the team and system
Expand Down Expand Up @@ -438,9 +441,3 @@ func parseVersionFile(r io.Reader) (int, error) {
}
return header.Version, nil
}

func readLine(r *bufio.Reader) ([]byte, error) {
line, err := r.ReadBytes('\n')
line = bytes.TrimSpace(line)
return line, err
}

0 comments on commit cfa38b8

Please sign in to comment.