Skip to content

Commit

Permalink
Major changes in memory usage (#301)
Browse files Browse the repository at this point in the history 
* Major changes in memory usage

Stops storing all read files in memory.
This change applies to regular files only,
not pipes or compressed files.

Change to control by channel to make
it work more asynchronously.

Change File Watch to Directory Watch
to react to file descriptor changes.

* Allow exec reload

Recreated Exec and deprecated ExecCommand.
Separate the control part.
ControlNonFile integrated into ControlFile.

* Add follow-name

Follow the file name even if the file descriptor changes.
  • Loading branch information
@noborus
noborus committed Mar 22, 2023
1 parent cb8b4e7 commit 22cb002
Show file tree
Hide file tree
Showing 16 changed files with 701 additions and 252 deletions.
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ require (
github.com/gdamore/tcell/v2 v2.6.0
github.com/hashicorp/golang-lru v0.5.4
github.com/jwalton/gchalk v1.3.0
github.com/klauspost/compress v1.16.0
github.com/klauspost/compress v1.16.3
github.com/mattn/go-runewidth v0.0.14
github.com/pierrec/lz4 v2.6.1+incompatible
github.com/rivo/uniseg v0.4.4
github.com/spf13/cobra v1.6.1
github.com/spf13/viper v1.15.0
github.com/ulikunitz/xz v0.5.11
golang.org/x/exp v0.0.0-20230304125523-9ff063c70017
golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0
golang.org/x/sync v0.1.0
golang.org/x/term v0.6.0
)
Expand All @@ -31,7 +31,7 @@ require (
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.7 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/spf13/afero v1.9.4 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
Expand Down
17 changes: 10 additions & 7 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,8 @@ github.com/jwalton/gchalk v1.3.0/go.mod h1:ytRlj60R9f7r53IAElbpq4lVuPOPNg2J4tJcC
github.com/jwalton/go-supportscolor v1.1.0 h1:HsXFJdMPjRUAx8cIW6g30hVSFYaxh9yRQwEWgkAR7lQ=
github.com/jwalton/go-supportscolor v1.1.0/go.mod h1:hFVUAZV2cWg+WFFC4v8pT2X/S2qUUBYMioBD9AINXGs=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
Expand Down Expand Up @@ -180,8 +180,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/afero v1.9.4 h1:Sd43wM1IWz/s1aVXdOBkjJvuP8UdyqioeE4AmM0QsBs=
github.com/spf13/afero v1.9.4/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM=
github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
Expand Down Expand Up @@ -225,7 +225,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
Expand All @@ -236,8 +236,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230304125523-9ff063c70017 h1:3Ea9SZLCB0aRIhSEjM+iaGIlzzeDJdpi579El/YIhEE=
golang.org/x/exp v0.0.0-20230304125523-9ff063c70017/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo=
golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
Expand Down Expand Up @@ -293,6 +293,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -350,6 +351,7 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand All @@ -373,6 +375,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
Expand Down
20 changes: 6 additions & 14 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@ package main
import (
"errors"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"

"github.com/noborus/ov/oviewer"
Expand Down Expand Up @@ -142,23 +140,14 @@ func ExecCommand(args []string) error {
if len(args) == 0 {
return ErrNoArgument
}

command := exec.Command(args[0], args[1:]...)
ov, err := oviewer.ExecCommand(command)
cmd := oviewer.NewCommand(args)
ov, err := cmd.Exec()
if err != nil {
return err
}

defer func() {
if command == nil || command.Process == nil {
return
}
if err := command.Process.Kill(); err != nil {
log.Println(err)
}
if err := command.Wait(); err != nil {
log.Println(err)
}
cmd.Wait()
}()

ov.SetConfig(config)
Expand Down Expand Up @@ -276,6 +265,9 @@ func init() {
rootCmd.PersistentFlags().BoolP("follow-section", "", false, "follow section")
_ = viper.BindPFlag("general.FollowSection", rootCmd.PersistentFlags().Lookup("follow-section"))

rootCmd.PersistentFlags().BoolP("follow-name", "", false, "follow name mode")
_ = viper.BindPFlag("general.FollowName", rootCmd.PersistentFlags().Lookup("follow-name"))

rootCmd.PersistentFlags().IntP("watch", "T", 0, "watch mode interval")
_ = viper.BindPFlag("general.WatchInterval", rootCmd.PersistentFlags().Lookup("watch"))

Expand Down
13 changes: 11 additions & 2 deletions oviewer/action.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,11 @@ func (root *Root) toggleFollowMode() {
// toggleFollowAll toggles follow all mode.
func (root *Root) toggleFollowAll() {
root.General.FollowAll = !root.General.FollowAll
root.mu.Lock()
for _, doc := range root.DocList {
doc.latestNum = doc.BufEndNum()
}
root.mu.Unlock()
}

// toggleFollowSection toggles follow section mode.
Expand All @@ -84,9 +89,11 @@ func (root *Root) closeFile() {
root.setMessage("already closed")
return
}
if err := root.Doc.close(); err != nil {
log.Printf("closeFile: %s", err)
if root.Doc.seekable {
root.setMessage("cannnot close")
return
}
root.Doc.closeControl()
root.setMessagef("close file %s", root.Doc.FileName)
log.Printf("close file %s", root.Doc.FileName)
}
Expand All @@ -98,10 +105,12 @@ func (root *Root) reload(m *Document) {
return
}

root.mu.Lock()
if err := m.reload(); err != nil {
log.Printf("cannot reload: %s", err)
return
}
root.mu.Unlock()
root.releaseEventBuffer()
// Reserve time to read.
time.Sleep(100 * time.Millisecond)
Expand Down
4 changes: 1 addition & 3 deletions oviewer/doclist.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,7 @@ func (root *Root) closeDocument() {
root.setMessagef("close [%d]%s", root.CurrentDoc, root.Doc.FileName)
log.Printf("close [%d]%s", root.CurrentDoc, root.Doc.FileName)
root.mu.Lock()
if err := root.DocList[root.CurrentDoc].close(); err != nil {
log.Printf("%s:%s", root.Doc.FileName, err)
}
root.DocList[root.CurrentDoc].closeControl()
root.DocList = append(root.DocList[:root.CurrentDoc], root.DocList[root.CurrentDoc+1:]...)
if root.CurrentDoc > 0 {
root.CurrentDoc--
Expand Down
85 changes: 61 additions & 24 deletions oviewer/document.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,16 @@ type Document struct {
FileName string
// Caption is an additional caption to display after the file name.
Caption string

// filepath stores the absolute pathname for file watching.
filepath string
// File is the os.File.
file *os.File

// chunks is the content of the file to be stored in chunks.
chunks []*chunk

// notify when eof is reached.
eofCh chan struct{}
// notify when reopening.
followCh chan struct{}

// notify when a file changes.
changCh chan struct{}
// Specifies the chunk to read. -1 reads the new last line.
ctlCh chan controlSpecifier

cancel context.CancelFunc

Expand All @@ -61,6 +57,7 @@ type Document struct {
// offset
offset int64

startNum int
// endNum is the number of the last line read.
endNum int

Expand Down Expand Up @@ -102,6 +99,9 @@ type Document struct {
// 1 if there is a closed.
closed int32

// 1 if there is a read cancel.
readCancel int32

// WatchMode is watch mode.
WatchMode bool
// preventReload is true to prevent reload.
Expand Down Expand Up @@ -130,16 +130,14 @@ type chunk struct {
// NewDocument returns Document.
func NewDocument() (*Document, error) {
m := &Document{
eofCh: make(chan struct{}),
followCh: make(chan struct{}),
changCh: make(chan struct{}),
tickerDone: make(chan struct{}),
general: general{
ColumnDelimiter: "",
TabWidth: 8,
MarkStyleWidth: 1,
PlainMode: false,
},
ctlCh: make(chan controlSpecifier),
seekable: true,
preventReload: false,
chunks: []*chunk{
Expand Down Expand Up @@ -189,7 +187,12 @@ func OpenDocument(fileName string) (*Document, error) {
m.seekable = false
}

if err := m.ReadFile(fileName); err != nil {
f, err := open(fileName)
if err != nil {
return nil, err
}
m.FileName = fileName
if err := m.ControlFile(f); err != nil {
return nil, err
}
return m, nil
Expand All @@ -204,34 +207,66 @@ func STDINDocument() (*Document, error) {

m.seekable = false
m.Caption = "(STDIN)"
if err := m.ReadFile(""); err != nil {
f, err := open("")
if err != nil {
return nil, err
}
if err := m.ControlFile(f); err != nil {
return nil, err
}
return m, nil
}

// GetLine returns one line from buffer.
func (m *Document) GetLine(n int) string {
m.mu.Lock()
defer m.mu.Unlock()

if n < 0 || n >= m.endNum {
if n < m.startNum || n >= m.endNum {
return ""
}

chunkNum := n / ChunkSize
chunkLine := n % ChunkSize
if len(m.chunks)-1 < chunkNum {
log.Println("over chunk size: ", chunkNum)
return ""
}
chunk := m.chunks[chunkNum]
if len(chunk.lines)-1 < chunkLine {
log.Printf("over lines size: chunk[%d]:%d < %d", chunkNum, len(chunk.lines)-1, chunkLine)
return ""

if len(chunk.lines) == 0 && atomic.LoadInt32(&m.closed) == 0 {
m.loadControl(chunkNum)
}

return chunk.lines[chunkLine]
m.mu.Lock()
defer m.mu.Unlock()

cn := n % ChunkSize
if cn < len(chunk.lines) {
return chunk.lines[cn]
}
log.Println("not load", n, m.endNum)
return ""
}

// loadControl sends instructions to load chunks into memory.
func (m *Document) loadControl(chunkNum int) {
sc := controlSpecifier{
control: loadControl,
chunkNum: chunkNum,
done: make(chan struct{}),
}
m.ctlCh <- sc
<-sc.done
}

func (m *Document) closeControl() {
atomic.StoreInt32(&m.readCancel, 1)
sc := controlSpecifier{
control: closeControl,
done: make(chan struct{}),
}

log.Println("close send")
m.ctlCh <- sc
<-sc.done
log.Println("receive done")
atomic.StoreInt32(&m.readCancel, 0)
}

// CurrentLN returns the currently displayed line number.
Expand Down Expand Up @@ -305,7 +340,9 @@ func (m *Document) getLineC(lN int, tabWidth int) (LineC, bool) {
str: str,
pos: pos,
}
m.cache.Add(lN, line)
if len(org) != 0 {
m.cache.Add(lN, line)
}

lc := make(contents, len(org))
copy(lc, org)
Expand Down
6 changes: 4 additions & 2 deletions oviewer/document_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ func TestDocument_lineToContents(t *testing.T) {
if err := m.ReadAll(bytes.NewBufferString(tt.str)); err != nil {
t.Fatal(err)
}
<-m.eofCh
for !m.BufEOF() {
}
t.Logf("num:%d", m.BufEndNum())
got, err := m.contents(tt.args.lN, tt.args.tabWidth)
if (err != nil) != tt.wantErr {
Expand Down Expand Up @@ -136,7 +137,8 @@ func TestDocument_Export(t *testing.T) {
t.Fatal(err)
}
w := &bytes.Buffer{}
<-m.eofCh
for !m.BufEOF() {
}
m.bottomLN = m.BufEndNum()
m.Export(w, tt.args.start, tt.args.end)
if gotW := w.String(); gotW != tt.wantW {
Expand Down
3 changes: 3 additions & 0 deletions oviewer/draw.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,9 @@ func (root *Root) normalLeftStatus() (contents, int) {
if root.General.FollowAll {
modeStatus = "(Follow All)"
}
if root.Doc.FollowName {
modeStatus = "(Follow Name)"
}
// Watch mode doubles as FollowSection mode.
if root.Doc.WatchMode {
modeStatus += "(Watch)"
Expand Down
Loading

0 comments on commit 22cb002

Please sign in to comment.