Is there any function or method that clears the cached memory during the process?

[luxiant]

Here I'm trying to run this function.

// channel function
func splittedDfProcess(dataframe dataframe.DataFrame, useModels customModel, c chan<- []sentimentRow) {
	var results []sentimentRow
	bar := progressbar.Default(int64(len(dataframe.Col("text").Records())))
	for i := 0; i < dataframe.Nrow(); i++ {
		index := series.Indexes([]int{i})
		row := dataframe.Subset(index)
		result := bertSentimentProcess(row, useModels)
		results = append(results, result)
		bar.Add(1)
	}
	c <- results
}

func bertSentimentProcess(dataframe dataframe.DataFrame, useModels customModel) sentimentRow {
	corpus := dataframe.Col("text").Records()[0]
	preprocessedCorpus := preprocessSentence(corpus)
	newSequence := tokenizer.NewInputSequence(preprocessedCorpus)
	encodeInput := tokenizer.NewSingleEncodeInput(newSequence)
	encodeCorpus, _ := useModels.tokenizer.Encode(encodeInput, true)

        // fit all encoding length into 128
	maxLength := 128
	ps := tokenizer.NewPaddingStrategy(tokenizer.WithFixed(maxLength))
	paddingParameter := tokenizer.PaddingParams{Strategy: *ps, Direction: 1, PadId: 1, PadTypeId: 0, PadToken: "[PAD]"}
	var finalEncode tokenizer.Encoding
	switch {
	case encodeCorpus.Len() > maxLength:
		truncatedEncode, _ := encodeCorpus.Truncate(maxLength, 2)
		finalEncode = *truncatedEncode
	case encodeCorpus.Len() < maxLength:
		var corpusConverted []tokenizer.Encoding
		corpusConverted = append(corpusConverted, *encodeCorpus)
		encodeList := tokenizer.PadEncodings(corpusConverted, paddingParameter)
		finalEncode = encodeList[0]
	default:
		finalEncode = *encodeCorpus
	}
	var tokInput []int64 = make([]int64, maxLength)
	for i := 0; i < len(finalEncode.Ids); i++ {
		tokInput[i] = int64(finalEncode.Ids[i])
	}

        // convert into input tensor and forward
	var idTurnIntoTensor []ts.Tensor
	idTurnIntoTensor = append(idTurnIntoTensor, *ts.TensorFrom(tokInput))
	idInputTensor := ts.MustStack(idTurnIntoTensor, 0).MustTo(gotch.CPU, true)
	var torchResult *ts.Tensor
	ts.NoGrad(func() {
		torchResult, _, _ = useModels.bertModel.ForwardT(idInputTensor, ts.None, ts.None, ts.None, ts.None, false)
	})

        // clone the torchResult (for memory saving?)
	torchResultCopy := torchResult.MustShallowClone()

        // get softmax value of each label
	categoryProb := torchResultCopy.MustSoftmax(-1, gotch.Float, true)
	var long, neutral, short float64
	var sentiment string
	probs := categoryProb.Float64Values()
	long = probs[0]
	neutral = probs[1]
	short = probs[2]
	if long > neutral && long > short {
		sentiment = "long"
	} else if neutral > long && neutral > short {
		sentiment = "neutral"
	} else {
		sentiment = "short"
	}
	return sentimentRow{
		post_num:  dataframe.Col("post_num").Records()[0],
		time:      dataframe.Col("time").Records()[0],
		text:      dataframe.Col("text").Records()[0],
		long:      long,
		neutral:   neutral,
		short:     short,
		sentiment: sentiment,
	}
}
...

I'm running my code in CPU in codespace with 32GB of RAM. This function kills the process because of out of memory. I tried to find a way to reduce memory consumption, especially emptying the cached memory after process of every iteration. I'm also considering another way, based on my experience in python that .eval() in pytorch helps reducing memory consumption. I have found that 'MustDrop()' in ts package drops the tensor, but I'm not sure that it will work. Am I in a right approach?

[sugarme]

@luxiant ,

We choose manual memory management in gotch meaning any derived/generated tensors must be manually deleted otherwise there's memory leak built up.

You can use MustDrop() method or optional delete if you chain your tensor operation. E.g x = x.MustAdd(y, true) where 'true' means deleting tensor x before reassigning adding result to x.

Also, to stop autograd accumulation in inference mode, wrap model forward as you did is the way to go.

Please see comment for potential leak below:

        // convert into input tensor and forward
	var idTurnIntoTensor []ts.Tensor
	idTurnIntoTensor = append(idTurnIntoTensor, *ts.TensorFrom(tokInput))
	idInputTensor := ts.MustStack(idTurnIntoTensor, 0).MustTo(gotch.CPU, true)
        // all original tensors must be deleted now as you just use the stacked tensor
        for _, x := range idTurnIntoTensor{
          x.MustDrop()
        }

	var torchResult *ts.Tensor
	ts.NoGrad(func() {
		// torchResult, _, _ = useModels.bertModel.ForwardT(idInputTensor, ts.None, ts.None, ts.None, ts.None, false)
		torchResult, x, y := useModels.bertModel.ForwardT(idInputTensor, ts.None, ts.None, ts.None, ts.None, false)
        // if x, y are tensors they must be explicitly deleted here
        x.MustDrop()
        y.MustDrop()
	})

        // clone the torchResult (for memory saving?) <- No. ShallowClone() just copy tensor. 
	torchResultCopy := torchResult.MustShallowClone() // <- `torchResult` may be a source of memory leak.

        // get softmax value of each label
	categoryProb := torchResultCopy.MustSoftmax(-1, gotch.Float, true)
	var long, neutral, short float64
	var sentiment string
	probs := categoryProb.Float64Values() // <- categoryProb needs to free either with optional `categoryProb.Float64Values(true) or in a separate statement `categoryProb.MustDrop()`
    categoryProb.MustDrop()
	long = probs[0]
	neutral = probs[1]
	short = probs[2]
	if long > neutral && long > short {
		sentiment = "long"
	} else if neutral > long && neutral > short {
		sentiment = "neutral"
	} else {
		sentiment = "short"
	}

Hope that helps.

[luxiant]

I thought I was almost done, but this is really frustrating.

Here is my whole code : https://github.com/luxiant/marketsentiment/blob/main/main.go

I tried to find the cause where memory leak is occuring. Here is the map of allocation.

I can see that the main part of the code, which is the process wrapped in ts.NoGrad, takes up a lot of memory during the process. I thought the memory allocation during the tensor calculation is dropped after I get an array of softmax probability, and the iteration is passing a pointer of the models, not a whole struct, after I checked another closed issues #76.

Is the normalizer the cause? If so, I need to move this question to tokenizer...

[sugarme]

@luxiant ,

Nice memory profile graph! In general, I would create a simple inference example on CPU device and run for-loop input through it to check for any memory leak inside the model.

Just have a quick look at your inference function and there a potential mem leak there:

func (m *customModel) bertSentimentProcess(dataframe dataframe.DataFrame) sentimentRow {
	var logit []float64
	ts.NoGrad(func() {
		torchResult, _, _ := m.bertModel.ForwardT(
                        // a new tensor is created by calling this function. Then no where you delete it.
			processSentenceIntoInput(m.tokenizer, dataframe.Col("text").Records()[0]), // <-- this is the leak
			ts.None,
			ts.None,
			ts.None,
			ts.None,
			false,
		)
		logit = torchResult.MustSoftmax(-1, gotch.Double, true).Float64Values(true)
	})
	var sentiment string
	switch {
	case logit[0] > logit[1] && logit[0] > logit[2]:
		sentiment = "long"
	case logit[1] > logit[0] && logit[1] > logit[2]:
		sentiment = "neutral"
	default:
		sentiment = "short"
	}
	return sentimentRow{
		post_num:  dataframe.Col("post_num").Records()[0],
		time:      dataframe.Col("time").Records()[0],
		text:      dataframe.Col("text").Records()[0],
		long:      logit[0],
		neutral:   logit[1],
		short:     logit[2],
		sentiment: sentiment,
	}
}

I would do in 2-step and delete the input after running forward:

func (m *customModel) bertSentimentProcess(dataframe dataframe.DataFrame) sentimentRow {
	var logit []float64
	ts.NoGrad(func() {
                // input is a newly created tensor
                input := processSentenceIntoInput(m.tokenizer, dataframe.Col("text").Records()[0])

                // Also, to avoid potential mem leak, I would receive all returned values if they have *ts.Tensor in them
                // then delete if existing.
		torchResult, unused1,  unsed2:= m.bertModel.ForwardT(
			input,
			ts.None,
			ts.None,
			ts.None,
			ts.None,
			false,
		)
                input.MustDrop() // <-- delete after using.
                // delete `unused1` and `unused2` here if they are not empty
		logit = torchResult.MustSoftmax(-1, gotch.Double, true).Float64Values(true)
	})
	var sentiment string
	switch {
	case logit[0] > logit[1] && logit[0] > logit[2]:
		sentiment = "long"
	case logit[1] > logit[0] && logit[1] > logit[2]:
		sentiment = "neutral"
	default:
		sentiment = "short"
	}
	return sentimentRow{
		post_num:  dataframe.Col("post_num").Records()[0],
		time:      dataframe.Col("time").Records()[0],
		text:      dataframe.Col("text").Records()[0],
		long:      logit[0],
		neutral:   logit[1],
		short:     logit[2],
		sentiment: sentiment,
	}
}

[luxiant]

I tried it already with following code during troubleshooting, thinking that is the most possible point. But I met segfault error when I receive the unused values from Forward method and then try to manually free them. That is why I chose not to receive the unnecessary values at the first place. But yes...I can tell that this was meaningless. The process allocates memory to these unnecessary tensor slices even though I'm not receiving the values.

Is there any idea for this? I can't even remember how many times I run into segfault error to deal with this and can't think of the way to go forward.

func (m *customModel) bertSentimentProcess(dataframe dataframe.DataFrame) sentimentRow {
	var logit []float64
	ts.NoGrad(func() {
                // input is a newly created tensor
                input := processSentenceIntoInput(m.tokenizer, dataframe.Col("text").Records()[0])

                // receive all values, and then delete unusued tensor slices if existing.
		torchResult, unused1, unused2:= m.bertModel.ForwardT(
			input,
			ts.None,
			ts.None,
			ts.None,
			ts.None,
			false,
		)
                // drop input tensor. this works.
                input.MustDrop()

                // trying to drop unused tensor slices
                for _, tensor := range unused1 {
                        (&tensor).MustDrop() // <- this causes segfault error
                }
                for _, tensor := range unused2 {
                        (&tensor).MustDrop() // <- this causes segfault error
                }
		logit = torchResult.MustSoftmax(-1, gotch.Double, true).Float64Values(true)
	})
	var sentiment string
	switch {
	case logit[0] > logit[1] && logit[0] > logit[2]:
		sentiment = "long"
	case logit[1] > logit[0] && logit[1] > logit[2]:
		sentiment = "neutral"
	default:
		sentiment = "short"
	}
	return sentimentRow{
		post_num:  dataframe.Col("post_num").Records()[0],
		time:      dataframe.Col("time").Records()[0],
		text:      dataframe.Col("text").Records()[0],
		long:      logit[0],
		neutral:   logit[1],
		short:     logit[2],
		sentiment: sentiment,
	}
}

[luxiant]

Oh, I think this should be on issue. I'll move on.