Made it possible to set a memory limit

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/viper v1.15.0
 	github.com/ulikunitz/xz v0.5.11
-	golang.org/x/exp v0.0.0-20230420155640-133eef4313cb
+	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
 	golang.org/x/sync v0.1.0
 	golang.org/x/term v0.7.0
 )

go.sum

@@ -237,8 +237,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk=
-golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
+golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o=
+golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

main.go

@@ -81,6 +81,18 @@ It supports various compressed files(gzip, bzip2, zstd, lz4, and xz).
 		oviewer.OverStrikeStyle = oviewer.ToTcellStyle(config.StyleOverStrike)
 		oviewer.OverLineStyle = oviewer.ToTcellStyle(config.StyleOverLine)
 
+		if config.FileLoadChunkLimit >= 0 {
+			if config.FileLoadChunkLimit < 2 {
+				config.FileLoadChunkLimit = 2
+			}
+			oviewer.FileLoadChunkLimit = config.FileLoadChunkLimit
+		}
+		if config.LoadChunkLimit >= 0 {
+			if config.LoadChunkLimit < 2 {
+				config.LoadChunkLimit = 2
+			}
+			oviewer.LoadChunkLimit = config.LoadChunkLimit
+		}
 		SetRedirect()
 
 		if execCommand {
@@ -308,6 +320,12 @@ func init() {
 	rootCmd.PersistentFlags().StringP("view-mode", "", "", "view mode")
 	_ = viper.BindPFlag("ViewMode", rootCmd.PersistentFlags().Lookup("view-mode"))
 
+	rootCmd.PersistentFlags().IntP("load-limit", "", -1, "Limit loading chunks")
+	_ = viper.BindPFlag("LoadChunkLimit", rootCmd.PersistentFlags().Lookup("load-limit"))
+
+	rootCmd.PersistentFlags().IntP("file-load-limit", "", 100, "Limit chunks loading files into memory")
+	_ = viper.BindPFlag("FileLoadChunkLimit", rootCmd.PersistentFlags().Lookup("file-load-limit"))
+
 	rootCmd.PersistentFlags().BoolP("debug", "", false, "debug mode")
 	_ = viper.BindPFlag("Debug", rootCmd.PersistentFlags().Lookup("debug"))
 }

oviewer/action.go

@@ -106,7 +106,7 @@ func (root *Root) closeFile() {
 		root.setMessage("cannot close")
 		return
 	}
-	root.Doc.closeControl()
+	root.Doc.requestClose()
 	root.setMessagef("close file %s", root.Doc.FileName)
 	log.Printf("close file %s", root.Doc.FileName)
 }

oviewer/config.go

@@ -3,6 +3,8 @@ package oviewer
 // NewConfig return the structure of Config with default values.
 func NewConfig() Config {
 	return Config{
+		LoadChunkLimit:     -1,
+		FileLoadChunkLimit: 100,
 		StyleHeader: OVStyle{
 			Bold: true,
 		},

oviewer/control.go

@@ -12,26 +12,32 @@ import (
 
 type controlSpecifier struct {
 	searcher Searcher
-	control  control
+	request  request
 	chunkNum int
 	done     chan bool
 }
 
-type control string
+type request string
 
 const (
-	firstControl    = "first"
-	continueControl = "read"
-	followControl   = "follow"
-	closeControl    = "close"
-	reloadControl   = "reload"
-	loadControl     = "load"
-	searchControl   = "search"
+	requestStart    = "start"
+	requestContinue = "read"
+	requestFollow   = "follow"
+	requestClose    = "close"
+	requestReload   = "reload"
+	requestLoad     = "load"
+	requestSearch   = "search"
 )
 
 // ControlFile controls file read and loads in chunks.
 // ControlFile can be reloaded by file name.
 func (m *Document) ControlFile(file *os.File) error {
+	if m.seekable {
+		m.loadedChunks.Resize(FileLoadChunkLimit + 1)
+	} else {
+		m.loadedChunks.Resize(LoadChunkLimit + 1)
+	}
+
 	go func() {
 		atomic.StoreInt32(&m.closed, 0)
 		r, err := m.fileReader(file)
@@ -44,7 +50,7 @@ func (m *Document) ControlFile(file *os.File) error {
 		for sc := range m.ctlCh {
 			reader, err = m.control(sc, reader)
 			if err != nil {
-				log.Println(sc.control, err)
+				log.Println(sc.request, err)
 			}
 			if sc.done != nil {
 				if err != nil {
@@ -58,52 +64,118 @@ func (m *Document) ControlFile(file *os.File) error {
 		log.Println("close m.ctlCh")
 	}()
 
-	m.startControl()
+	m.requestStart()
 	return nil
 }
 
-func (m *Document) control(sc controlSpecifier, reader *bufio.Reader) (*bufio.Reader, error) {
-	if atomic.LoadInt32(&m.closed) == 1 && sc.control != reloadControl {
-		return nil, fmt.Errorf("%w %s", ErrAlreadyClose, sc.control)
+func (m *Document) mangesChunksFile(chunkNum int) error {
+	for m.loadedChunks.Len() > FileLoadChunkLimit {
+		k, _, _ := m.loadedChunks.RemoveOldest()
+		if chunkNum != k {
+			log.Println("remove chunk", k)
+			m.chunks[k].lines = nil
+		}
+	}
+
+	chunk := m.chunks[chunkNum]
+	if len(chunk.lines) != 0 || atomic.LoadInt32(&m.closed) != 0 {
+		return fmt.Errorf("no need to add chunk %d", chunkNum)
+	}
+	if chunkNum != 0 {
+		m.loadedChunks.Add(chunkNum, struct{}{})
+	}
+	return nil
+}
+
+func (m *Document) mangesChunksMem(chunkNum int) error {
+	m.currentChunk = chunkNum
+	maxChunk := len(m.chunks)
+	if m.currentChunk < maxChunk-1 {
+		return fmt.Errorf("no need to add chunk %d", chunkNum)
 	}
+	if (LoadChunkLimit < 0) || (m.loadedChunks.Len() < LoadChunkLimit) {
+		return nil
+	}
+	k, _, _ := m.loadedChunks.GetOldest()
+	log.Println("remove chunk", k, m.loadedChunks.Len(), LoadChunkLimit)
+	m.loadedChunks.Remove(k)
+	m.chunks[k].lines = nil
+	m.mu.Lock()
+	m.startNum = (k + 1) * ChunkSize
+	m.mu.Unlock()
+	return nil
+}
+
+func (m *Document) canRead() bool {
+	if LoadChunkLimit > 0 && m.loadedChunks.Len() >= LoadChunkLimit {
+		return false
+	}
+	return true
+}
 
+func (m *Document) control(sc controlSpecifier, reader *bufio.Reader) (*bufio.Reader, error) {
+	if atomic.LoadInt32(&m.closed) == 1 && sc.request != requestReload {
+		return nil, fmt.Errorf("%w %s", ErrAlreadyClose, sc.request)
+	}
 	var err error
-	switch sc.control {
-	case firstControl:
+	switch sc.request {
+	case requestStart:
 		reader, err = m.firstRead(reader)
 		if !m.BufEOF() {
-			m.continueControl()
+			m.requestContinue()
 		}
 		return reader, err
-	case continueControl:
+	case requestContinue:
+		if !m.seekable {
+			if !m.canRead() {
+				log.Println("stop read", m.loadedChunks.Len(), FileLoadChunkLimit)
+				return reader, nil
+			}
+		}
 		reader, err = m.continueRead(reader)
 		if !m.BufEOF() {
-			m.continueControl()
+			m.requestContinue()
 		}
 		return reader, err
-	case followControl:
+	case requestFollow:
 		return m.followRead(reader)
-	case loadControl:
-		return m.readChunk(reader, sc.chunkNum)
-	case searchControl:
+	case requestLoad:
+		if m.seekable {
+			if err := m.mangesChunksFile(sc.chunkNum); err != nil {
+				return reader, nil
+			}
+			return m.readChunk(reader, sc.chunkNum)
+		}
+
+		if !m.BufEOF() {
+			if err := m.mangesChunksMem(sc.chunkNum); err != nil {
+				return reader, nil
+			}
+			m.requestContinue()
+		}
+		return reader, nil
+	case requestSearch:
 		_, err = m.searchChunk(sc.chunkNum, sc.searcher)
 		if err != nil {
 			if errors.Is(err, ErrNotFound) {
 				return reader, nil
 			}
 			return reader, err
 		}
+		if err := m.mangesChunksFile(sc.chunkNum); err != nil {
+			return reader, nil
+		}
 		return m.readChunk(reader, sc.chunkNum)
-	case reloadControl:
+	case requestReload:
 		reader, err = m.reloadRead(reader)
-		m.startControl()
+		m.requestStart()
 		return reader, err
-	case closeControl:
+	case requestClose:
 		err = m.close()
 		log.Println(err)
 		return reader, err
 	default:
-		panic(fmt.Sprintf("unexpected %s", sc.control))
+		panic(fmt.Sprintf("unexpected %s", sc.request))
 	}
 }
 
@@ -112,11 +184,12 @@ func (m *Document) control(sc controlSpecifier, reader *bufio.Reader) (*bufio.Re
 func (m *Document) ControlLog() error {
 	go func() {
 		for sc := range m.ctlCh {
-			switch sc.control {
-			case reloadControl:
+			switch sc.request {
+			case requestLoad:
+			case requestReload:
 				m.reset()
 			default:
-				panic(fmt.Sprintf("unexpected %s", sc.control))
+				panic(fmt.Sprintf("unexpected %s", sc.request))
 			}
 			if sc.done != nil {
 				close(sc.done)
@@ -134,25 +207,30 @@ func (m *Document) ControlReader(r io.Reader, reload func() *bufio.Reader) error
 	go func() {
 		var err error
 		for sc := range m.ctlCh {
-			switch sc.control {
-			case firstControl:
+			switch sc.request {
+			case requestStart:
 				reader, err = m.firstRead(reader)
 				if !m.BufEOF() {
-					m.continueControl()
+					m.requestContinue()
 				}
-			case continueControl:
+			case requestContinue:
 				reader, err = m.continueRead(reader)
 				if !m.BufEOF() {
-					m.continueControl()
+					m.requestContinue()
+				}
+			case requestLoad:
+				m.currentChunk = sc.chunkNum
+				if sc.chunkNum != 0 {
+					m.loadedChunks.Add(sc.chunkNum, struct{}{})
 				}
-			case reloadControl:
+			case requestReload:
 				if reload != nil {
 					log.Println("reset")
 					reader = reload()
-					m.startControl()
+					m.requestStart()
 				}
 			default:
-				panic(fmt.Sprintf("unexpected %s", sc.control))
+				panic(fmt.Sprintf("unexpected %s", sc.request))
 			}
 			if err != nil {
 				log.Println(err)
@@ -164,22 +242,61 @@ func (m *Document) ControlReader(r io.Reader, reload func() *bufio.Reader) error
 		log.Println("close ctlCh")
 	}()
 
-	m.startControl()
+	m.requestStart()
 	return nil
 }
 
-func (m *Document) startControl() {
+// requestStart send instructions to start reading.
+func (m *Document) requestStart() {
 	go func() {
 		m.ctlCh <- controlSpecifier{
-			control: firstControl,
+			request: requestStart,
 		}
 	}()
 }
 
-func (m *Document) continueControl() {
+// requestContinue sends instructions to continue reading.
+func (m *Document) requestContinue() {
 	go func() {
 		m.ctlCh <- controlSpecifier{
-			control: continueControl,
+			request: requestContinue,
 		}
 	}()
 }
+
+// requestLoad sends instructions to load chunks into memory.
+func (m *Document) requestLoad(chunkNum int) {
+	sc := controlSpecifier{
+		request:  requestLoad,
+		chunkNum: chunkNum,
+		done:     make(chan bool),
+	}
+	m.ctlCh <- sc
+	<-sc.done
+}
+
+// requestSearch sends instructions to load chunks into memory.
+func (m *Document) requestSearch(chunkNum int, searcher Searcher) bool {
+	sc := controlSpecifier{
+		request:  requestSearch,
+		searcher: searcher,
+		chunkNum: chunkNum,
+		done:     make(chan bool),
+	}
+	m.ctlCh <- sc
+	return <-sc.done
+}
+
+// requestClose sends instructions to close the file.
+func (m *Document) requestClose() {
+	atomic.StoreInt32(&m.readCancel, 1)
+	sc := controlSpecifier{
+		request: requestClose,
+		done:    make(chan bool),
+	}
+
+	log.Println("close send")
+	m.ctlCh <- sc
+	<-sc.done
+	atomic.StoreInt32(&m.readCancel, 0)
+}

oviewer/doclist.go

@@ -50,7 +50,7 @@ func (root *Root) closeDocument() {
 	root.setMessagef("close [%d]%s", root.CurrentDoc, root.Doc.FileName)
 	log.Printf("close [%d]%s", root.CurrentDoc, root.Doc.FileName)
 	root.mu.Lock()
-	root.DocList[root.CurrentDoc].closeControl()
+	root.DocList[root.CurrentDoc].requestClose()
 	root.DocList = append(root.DocList[:root.CurrentDoc], root.DocList[root.CurrentDoc+1:]...)
 	if root.CurrentDoc > 0 {
 		root.CurrentDoc--