Skip to content

Commit

Permalink
测试 V2 字幕时间轴校正功能
Browse files Browse the repository at this point in the history
Signed-off-by: allan716 <525223688@qq.com>
  • Loading branch information
allanpk716 committed Nov 17, 2021
1 parent 9d2f8d6 commit a0543a1
Show file tree
Hide file tree
Showing 12 changed files with 419 additions and 100 deletions.
3 changes: 1 addition & 2 deletions go.mod
Expand Up @@ -9,6 +9,7 @@ require (
github.com/abadojack/whatlanggo v1.0.1
github.com/allanpk716/fake-useragent v0.2.1
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394
github.com/baabaaox/go-webrtcvad v1.0.1
github.com/beevik/etree v1.1.0
github.com/bodgit/sevenzip v1.1.0
github.com/emirpasic/gods v1.12.0
Expand Down Expand Up @@ -69,10 +70,8 @@ require (
github.com/andybalholm/brotli v1.0.0 // indirect
github.com/andybalholm/cascadia v1.2.0 // indirect
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 // indirect
github.com/baabaaox/go-webrtcvad v1.0.1 // indirect
github.com/bodgit/plumbing v1.1.0 // indirect
github.com/bodgit/windows v1.0.0 // indirect
github.com/brettbuddin/fourier v0.1.1 // indirect
github.com/connesc/cipherio v0.2.1 // indirect
github.com/dsnet/compress v0.0.1 // indirect
github.com/golang/snappy v0.0.3 // indirect
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Expand Up @@ -65,8 +65,6 @@ github.com/bodgit/sevenzip v1.1.0/go.mod h1:vRCJlX/FVjbcwUG9lyX1YQPbQA4Xxw/7puzc
github.com/bodgit/windows v1.0.0 h1:rLQ/XjsleZvx4fR1tB/UxQrK+SJ2OFHzfPjLWWOhDIA=
github.com/bodgit/windows v1.0.0/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/brettbuddin/fourier v0.1.1 h1:jjlYBAV018g3B2VH5wGI/XgZE9veEHXS7MmPa4BJgmo=
github.com/brettbuddin/fourier v0.1.1/go.mod h1:o4lEcyAaNBt8evqo2VXTSpRnZTts9lyzt73uRFtmD70=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
Expand Down
@@ -1,4 +1,4 @@
package sub_timeline_fixer
package calculate_curve_correlation

import "math"

Expand Down
@@ -1,4 +1,4 @@
package sub_timeline_fixer
package calculate_curve_correlation

import "testing"

Expand Down
149 changes: 149 additions & 0 deletions internal/pkg/dtw/fast_dtw.go
@@ -0,0 +1,149 @@
package dtw

import (
"container/list"
"math"
)

type T = float64
type TimeSeries = []T

/*
FastDTW
https://github.com/Citing/fastDTW
*/
func FastDTW(seriesX TimeSeries, seriesY TimeSeries, radius int) (distance T, path [][2]int) {
var minTSsize = radius + 2
if len(seriesX) < minTSsize || len(seriesY) < minTSsize {
distance, path = DTW(seriesX, seriesY, nil)
} else {
var shrunkX = reduceByHalf(seriesX)
var shrunkY = reduceByHalf(seriesY)
var _, lowResPath = FastDTW(shrunkX, shrunkY, radius)
var window = expandResWindow(lowResPath, len(seriesX), len(seriesY), radius)
distance, path = DTW(seriesX, seriesY, window)
}
return
}

func DTW(seriesX TimeSeries, seriesY TimeSeries, window [][2]int) (T, [][2]int) {
if window == nil {
window = make([][2]int, (len(seriesX)+1)*(len(seriesY)+1))
k := 0
for i := 0; i <= len(seriesX); i++ {
for j := 0; j <= len(seriesY); j++ {
window[k] = [2]int{i, j}
k++
}
}
} else {
for k := 0; k < len(window); k++ {
window[k] = [2]int{window[k][0] + 1, window[k][1] + 1}
}
}
type p struct {
dist T
neighborX int
neighborY int
}
var D = make(map[[2]int]p)
for _, v := range window {
// T type!
D[v] = p{math.MaxFloat64, 0, 0}
}
D[[2]int{0, 0}] = p{0, 0, 0}
for i := 1; i <= len(seriesX); i++ {
for j := 1; j <= len(seriesY); j++ {
v := [2]int{i, j}
dt := math.Abs(seriesX[v[0]-1] - seriesY[v[1]-1])
D[v] = func(p1 p, p2 p, p3 p) p {
var tmp p
if p1.dist < p2.dist {
tmp = p1
} else {
tmp = p2
}
if p3.dist < tmp.dist {
tmp = p3
}
return tmp
}(p{D[[2]int{v[0] - 1, v[1]}].dist + dt, v[0] - 1, v[1]}, p{D[[2]int{v[0], v[1] - 1}].dist + dt, v[0], v[1] - 1}, p{D[[2]int{v[0] - 1, v[1] - 1}].dist + dt, v[0] - 1, v[1] - 1})
}
}

var path_ list.List
for i, j := len(seriesX), len(seriesY); i != 0 || j != 0; {
path_.PushBack([2]int{i - 1, j - 1})
i, j = D[[2]int{i, j}].neighborX, D[[2]int{i, j}].neighborY
}
path := make([][2]int, path_.Len())
for i, j, k := len(seriesX), len(seriesY), 0; i != 0 || j != 0; k++ {
path[k] = ([2]int{i - 1, j - 1})
i, j = D[[2]int{i, j}].neighborX, D[[2]int{i, j}].neighborY
}

distance := D[[2]int{len(seriesX), len(seriesY)}].dist
return distance, path
}

func reduceByHalf(series TimeSeries) TimeSeries {
shrunk := make([]T, len(series)/2)
for i := 0; i < len(shrunk); i++ {
shrunk[i] = (series[2*i] + series[2*i+1]) / 2
}
return shrunk
}

func expandResWindow(path [][2]int, X int, Y int, radius int) [][2]int {
var window_ = make(map[[2]int]int)

for k := 0; k < len(path); k++ {
for i := 0; i <= 2*radius; i++ {
for j := 0; j <= 2*radius; j++ {
x := 2 * (path[k][0] - radius + i)
y := 2 * (path[k][1] - radius + j)
window_[[2]int{x, y}] = 1
window_[[2]int{x + 1, y}] = 1
window_[[2]int{x, y + 1}] = 1
window_[[2]int{x + 1, y + 1}] = 1
}
}
}

var window__ = make(map[[2]int]int)
start_j := 0
for i := 0; i < X; i++ {
new_start_j := -2
for j := start_j; j < Y; j++ {
if window_[[2]int{i, j}] == 1 {
window__[[2]int{i, j}] = 1
if new_start_j == -2 {
new_start_j = j
}
} else if new_start_j != -2 {
break
}
start_j = new_start_j
}
}

var window = make([][2]int, len(window__))
start_j = 0
k := 0
for i := 0; i < X; i++ {
new_start_j := -2
for j := start_j; j < Y; j++ {
if window_[[2]int{i, j}] == 1 {
window[k] = [2]int{i, j}
k++
if new_start_j == -2 {
new_start_j = j
}
} else if new_start_j != -2 {
break
}
start_j = new_start_j
}
}
return window
}
16 changes: 16 additions & 0 deletions internal/pkg/dtw/fast_dtw_test.go
@@ -0,0 +1,16 @@
package dtw

import (
"fmt"
"testing"
)

func TestFastDTW(t *testing.T) {
var balance1 = TimeSeries{1, 2, 3, 4, 5}
var balance2 = TimeSeries{1, 1.3, 2, 3, 5, 6}
//var balance2 = TimeSeries{1, 2, 3.1, 4, 5}
d, b := FastDTW(balance1, balance2, 1)

fmt.Println(d)
fmt.Println(b)
}
22 changes: 14 additions & 8 deletions internal/pkg/sub_helper/sub_helper.go
Expand Up @@ -354,7 +354,7 @@ func DeleteOneSeasonSubCacheFolder(seriesDir string) error {

/*
只针对英文字幕进行合并分散的 Dialogues
会遇到这样的字幕,如下
会遇到这样的字幕,如下0
2line-The Card Counter (2021) WEBDL-1080p.chinese(inside).ass
它的对白一句话分了两个 dialogue 去做。这样做后续字幕时间轴校正就会遇到问题,因为只有一半,匹配占比会很低
(每一个 Dialogue 的首字母需要分析,大写和小写的占比是多少,统计一下,正常的,和上述特殊的)
Expand All @@ -369,20 +369,21 @@ func MergeMultiDialogue4EngSubtitle(inSubParser *subparser.FileInfo) {
inSubParser.DialoguesEx = merger.Get()
}

// GetVADINfoFromSub 跟下面的 GetVADINfoFromSubNeedOffsetTime 函数功能一致
func GetVADINfoFromSub(infoSrc *subparser.FileInfo, FrontAndEndPer float64, SubUnitMaxCount int) ([]SubUnit, error) {
return GetVADINfoFromSubNeedOffsetTime(infoSrc, FrontAndEndPer, SubUnitMaxCount, 0)
// GetVADINfoFromSub 跟下面的 GetVADINfoFromSubNeedOffsetTimeWillInsert 函数功能一致
func GetVADINfoFromSub(infoSrc *subparser.FileInfo, FrontAndEndPer float64, SubUnitMaxCount int, insert bool) ([]SubUnit, error) {

return GetVADINfoFromSubNeedOffsetTimeWillInsert(infoSrc, FrontAndEndPer, SubUnitMaxCount, 0, insert)
}

/*
GetVADINfoFromSubNeedOffsetTime 只不过这里可以加一个每一句话固定的偏移时间
GetVADINfoFromSubNeedOffsetTimeWillInsert 只不过这里可以加一个每一句话固定的偏移时间
这里的字幕要求是完整的一个字幕
1. 抽取字幕的时间片段的时候,暂定,前 15% 和后 15% 要避开,前奏、主题曲、结尾曲
2. 将整个字幕,抽取连续 5 句对话为一个单元,提取时间片段信息
3. 可能还有一个需求,默认的模式是每五句话一个单元,还有一种模式是每一句话向后找到连续的四句话组成一个单元,允许重叠
目前看到的情况是前者的抽样率太低,需要使用后者的逻辑
*/
func GetVADINfoFromSubNeedOffsetTime(infoSrc *subparser.FileInfo, FrontAndEndPer float64, SubUnitMaxCount int, offsetTime float64) ([]SubUnit, error) {
func GetVADINfoFromSubNeedOffsetTimeWillInsert(infoSrc *subparser.FileInfo, FrontAndEndPer float64, SubUnitMaxCount int, offsetTime float64, insert bool) ([]SubUnit, error) {
if SubUnitMaxCount < 0 {
SubUnitMaxCount = 0
}
Expand Down Expand Up @@ -426,13 +427,18 @@ func GetVADINfoFromSubNeedOffsetTime(infoSrc *subparser.FileInfo, FrontAndEndPer
oneDialogueExTimeStart = oneDialogueExTimeStart.Add(offsetTimeDuration)
oneDialogueExTimeEnd = oneDialogueExTimeEnd.Add(offsetTimeDuration)
// 如果没有偏移就是 0
srcOneSubUnit.AddAndInsert(oneDialogueExTimeStart, oneDialogueExTimeEnd)
if insert == true {
srcOneSubUnit.AddAndInsert(oneDialogueExTimeStart, oneDialogueExTimeEnd)
} else {
srcOneSubUnit.Add(oneDialogueExTimeStart, oneDialogueExTimeEnd)
}
} else {
srcSubUnitList = append(srcSubUnitList, *srcOneSubUnit)
srcOneSubUnit = NewSubUnit()
// TODO 这里决定了插入数据的密度,有待测试
//i = i - SubUnitMaxCount + SubUnitMaxCount/5
i = i - SubUnitMaxCount + SubUnitMaxCount/2
//i = i - SubUnitMaxCount + SubUnitMaxCount/2
i = i - SubUnitMaxCount
}
}
if srcOneSubUnit.GetDialogueCount() > 0 {
Expand Down
75 changes: 71 additions & 4 deletions internal/pkg/sub_helper/sub_unit.go
@@ -1,10 +1,12 @@
package sub_helper

import (
"bufio"
"fmt"
"github.com/allanpk716/ChineseSubFinder/internal/pkg/my_util"
"github.com/allanpk716/ChineseSubFinder/internal/pkg/vad"
"math"
"os"
"time"
)

Expand All @@ -21,13 +23,33 @@ type SubUnit struct {

func NewSubUnit() *SubUnit {
return &SubUnit{
VADList: make([]vad.VADInfo, 0),
subCount: 0,
firstAdd: false,
outVADBytes: make([]byte, 0),
VADList: make([]vad.VADInfo, 0),
subCount: 0,
firstAdd: false,
outVADBytes: make([]byte, 0),
outVADFloats: make([]float64, 0),
}
}

func (s *SubUnit) Add(oneSubStartTime, oneSubEndTime time.Time) {

if s.firstAdd == false {
// 第一次 Add 需要给 baseTime 赋值
s.baseTime = oneSubStartTime
s.offsetStartTime = s.RealTimeToOffsetTime(oneSubStartTime)
s.firstAdd = true
}

s.offsetEndTime = oneSubEndTime.Add(-my_util.Time2Duration(s.baseTime))

// 添加 Start
s.VADList = append(s.VADList, *vad.NewVADInfoBase(true, time.Duration((my_util.Time2SecendNumber(oneSubStartTime))*math.Pow10(9))))
// 添加 End
s.VADList = append(s.VADList, *vad.NewVADInfoBase(false, time.Duration((my_util.Time2SecendNumber(oneSubEndTime))*math.Pow10(9))))

s.subCount++
}

// AddAndInsert 添加一句对白进来,并且填充中间的空白,间隔 10ms。传入的时间是真实的时间
func (s *SubUnit) AddAndInsert(oneSubStartTime, oneSubEndTime time.Time) {

Expand Down Expand Up @@ -247,4 +269,49 @@ func (s SubUnit) RealTimeToOffsetTime(realTime time.Time) time.Time {
return realTime.Add(-dd)
}

// Save2Txt 导出为 float64 的内容
func (s SubUnit) Save2Txt(outFileFPath string) error {

file, err := os.OpenFile(outFileFPath, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
return err
}
defer file.Close()
//写入文件时,使用带缓存的 *Writer
write := bufio.NewWriter(file)
for i := 0; i < len(s.VADList); i++ {
active := 0.0
if s.VADList[i].Active == true {
active = 1.0
}
_, err = write.WriteString(fmt.Sprintf("%v\n", active))
if err != nil {
return err
}
}
err = write.Flush()
if err != nil {
return err
}

return nil
}

// IsMatchKey 是否符合“钥匙”的标准
// features 是至少多少个“凹坑”
func (s SubUnit) IsMatchKey(features int) bool {
nowCount := 0
for _, value := range s.GetVADByteSlice() {
if value == 0 {
nowCount++
}
}

if nowCount >= features {
return true
}

return false
}

const perWindows = float64(vad.FrameDuration) / 1000
9 changes: 9 additions & 0 deletions internal/pkg/sub_timeline_fixer/fix_result.go
@@ -0,0 +1,9 @@
package sub_timeline_fixer

type FixResult struct {
OldMean float64
OldSD float64
NewMean float64
NewSD float64
Per float64 // 占比
}

0 comments on commit a0543a1

Please sign in to comment.