Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Vinai/refactor docs #1210

Merged
merged 30 commits into from
Jan 25, 2021
Merged
Show file tree
Hide file tree
Changes from 26 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
9f3913f
Prelimm methods of drw
VinaiRachakonda Jan 11, 2021
e121d06
Remove the some DocsReadWriter methods and move towards use of GetDoc…
VinaiRachakonda Jan 12, 2021
6f30212
Fix bug with repos that have no docs initialized
VinaiRachakonda Jan 12, 2021
a0b4235
clean up method
VinaiRachakonda Jan 12, 2021
bf91136
Formatting, comments and remove method:
VinaiRachakonda Jan 12, 2021
cd85e71
Commit compiles and passes docs test. Factors out much of the root up…
VinaiRachakonda Jan 12, 2021
3ef5011
Dump a bunch of methods environment.go
VinaiRachakonda Jan 12, 2021
a41b41d
Merge branch 'master' into vinai/refactor-docs
VinaiRachakonda Jan 12, 2021
efaa0bb
Final updates
VinaiRachakonda Jan 12, 2021
d380a2d
Remove a bunch of stuff from root_val
VinaiRachakonda Jan 14, 2021
3598fc5
Move some more methods areound and create a docs table package. Focus…
VinaiRachakonda Jan 14, 2021
3f2db74
Redo diffs.go to not use docs
VinaiRachakonda Jan 15, 2021
4160af0
Redo doc diffs, still failing tests
VinaiRachakonda Jan 15, 2021
28bd7db
Passing all docs tests...
VinaiRachakonda Jan 15, 2021
5ac7392
First step in refactoring out doltdocs package...
VinaiRachakonda Jan 15, 2021
4ad5617
type formatting and other things
VinaiRachakonda Jan 15, 2021
c4aa8f5
I think everything is passing
VinaiRachakonda Jan 15, 2021
ee97d12
Undo bats rewrite
VinaiRachakonda Jan 15, 2021
c9c684d
Small fixes
VinaiRachakonda Jan 15, 2021
e6f05fd
More changes
VinaiRachakonda Jan 19, 2021
ad439c8
repo fmt
VinaiRachakonda Jan 19, 2021
ccdfb0f
Pr review comments before large drw refactor
VinaiRachakonda Jan 21, 2021
9443ca6
Iniital changes
VinaiRachakonda Jan 21, 2021
531b88a
Method changes
VinaiRachakonda Jan 21, 2021
a345da2
additional changes
VinaiRachakonda Jan 21, 2021
76e3f1f
some refactoring :)
VinaiRachakonda Jan 22, 2021
b4a329e
rewrite the diffs alg
VinaiRachakonda Jan 23, 2021
288f01b
Make significant refacotring changes. Need to rewrite cli diff
VinaiRachakonda Jan 23, 2021
5ecabe6
More changes
VinaiRachakonda Jan 24, 2021
69356ca
Fix bc bug
VinaiRachakonda Jan 25, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion bats/docs.bats
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ teardown() {
teardown_common
}

@test "dolt status and ls to view valid docs on dolt init" {
@test "dolt status and ls to view supported docs on dolt init" {
echo license-text > LICENSE.md
echo readme-text > README.md
run ls
Expand Down
5 changes: 3 additions & 2 deletions go/cmd/dolt/commands/checkout.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/dolthub/dolt/go/cmd/dolt/errhand"
eventsapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi/v1alpha1"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdocs"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
Expand Down Expand Up @@ -121,7 +122,7 @@ func (cmd CheckoutCmd) Exec(ctx context.Context, commandStr string, args []strin
return HandleVErrAndExitCode(verr, usagePrt)
}

tbls, docs, err := actions.GetTblsAndDocDetails(dEnv.DocsReadWriter(), args)
tbls, docs, err := actions.GetTablesOrDocs(dEnv.DocsReadWriter(), args)
if err != nil {
verr := errhand.BuildDError("error: unable to parse arguments.").AddCause(err).Build()
return HandleVErrAndExitCode(verr, usagePrt)
Expand Down Expand Up @@ -189,7 +190,7 @@ func checkoutNewBranch(ctx context.Context, dEnv *env.DoltEnv, newBranch string,
return checkoutBranch(ctx, dEnv, newBranch)
}

func checkoutTablesAndDocs(ctx context.Context, dEnv *env.DoltEnv, tables []string, docs []doltdb.DocDetails) errhand.VerboseError {
func checkoutTablesAndDocs(ctx context.Context, dEnv *env.DoltEnv, tables []string, docs doltdocs.Docs) errhand.VerboseError {
err := actions.CheckoutTablesAndDocs(ctx, dEnv, tables, docs)

if err != nil {
Expand Down
82 changes: 55 additions & 27 deletions go/cmd/dolt/commands/diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
eventsapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi/v1alpha1"
"github.com/dolthub/dolt/go/libraries/doltcore/diff"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdocs"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
Expand Down Expand Up @@ -282,7 +283,21 @@ func parseDiffArgs(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPar

func getDiffRoots(ctx context.Context, dEnv *env.DoltEnv, args []string) (from, to *doltdb.RootValue, leftover []string, err error) {
headRoot, err := dEnv.StagedRoot(ctx)
workingRoot, err := dEnv.WorkingRootWithDocs(ctx)
if err != nil {
return nil, nil, nil, err
}

workingRoot, err := dEnv.WorkingRoot(ctx)
if err != nil {
return nil, nil, nil, err
}

docs, err := dEnv.DocsReadWriter().GetDocsOnDisk()
if err != nil {
return nil, nil, nil, err
}

workingRoot, err = env.UpdateRootWithDocs(ctx, workingRoot, docs)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This shouldn't be in the env package. Put it in the doltdocs or similar package.

if err != nil {
return nil, nil, nil, err
}
Expand Down Expand Up @@ -874,7 +889,7 @@ func createSplitter(fromSch schema.Schema, toSch schema.Schema, joiner *rowconv.
}

func diffDoltDocs(ctx context.Context, dEnv *env.DoltEnv, from, to *doltdb.RootValue, dArgs *diffArgs) error {
_, docDetails, err := actions.GetTblsAndDocDetails(dEnv.DocsReadWriter(), dArgs.docSet.AsSlice())
_, docs, err := actions.GetTablesOrDocs(dEnv.DocsReadWriter(), dArgs.docSet.AsSlice())

if err != nil {
return err
Expand All @@ -892,41 +907,54 @@ func diffDoltDocs(ctx context.Context, dEnv *env.DoltEnv, from, to *doltdb.RootV
return err
}

printDocDiffs(ctx, dEnv, fromDocTable, toDocTable, docDetails)
return nil
return printDocDiffs(ctx, fromDocTable, toDocTable, docs)
}

func printDocDiffs(ctx context.Context, dEnv *env.DoltEnv, fromTbl, toTbl *doltdb.Table, docDetails []doltdb.DocDetails) {
func printDocDiffs(ctx context.Context, fromTbl, toTbl *doltdb.Table, currentDocs doltdocs.Docs) error {
bold := color.New(color.Bold)

if docDetails == nil {
docDetails, _ = dEnv.GetAllValidDocDetails()
}
for _, doc := range currentDocs {
if doc.Text != nil && fromTbl == nil {
printAddedDoc(bold, doc.DocPk)
} else if fromTbl != nil {
if toTbl != nil {
sch1, err := toTbl.GetSchema(ctx)
if err != nil {
return err
}

docText, err := doltdocs.GetDocTextFromTbl(ctx, toTbl, &sch1, doc.DocPk)
if err != nil {
return err
}
doc.Text = docText
}

for _, doc := range docDetails {
if toTbl != nil {
sch1, _ := toTbl.GetSchema(ctx)
doc, _ = doltdb.AddNewerTextToDocFromTbl(ctx, toTbl, &sch1, doc)
sch2, err := fromTbl.GetSchema(ctx)
if err != nil {
return err
}

}
if fromTbl != nil {
sch2, _ := fromTbl.GetSchema(ctx)
doc, _ = doltdb.AddValueToDocFromTbl(ctx, fromTbl, &sch2, doc)
}
olderText, err := doltdocs.GetDocTextFromTbl(ctx, fromTbl, &sch2, doc.DocPk)
if err != nil {
return err
}

if olderText != nil {
newer := string(doc.Text)
older := string(olderText)
lines := textdiff.LineDiffAsLines(older, newer)

if doc.Value != nil {
newer := string(doc.NewerText)
older, _ := strconv.Unquote(doc.Value.HumanReadableString())
lines := textdiff.LineDiffAsLines(older, newer)
if doc.NewerText == nil {
printDeletedDoc(bold, doc.DocPk, lines)
} else if len(lines) > 0 && newer != older {
printModifiedDoc(bold, doc.DocPk, lines)
if doc.Text == nil {
printDeletedDoc(bold, doc.DocPk, lines)
} else if len(lines) > 0 && newer != older {
printModifiedDoc(bold, doc.DocPk, lines)
}
}
} else if doc.Value == nil && doc.NewerText != nil {
printAddedDoc(bold, doc.DocPk)
}
}

return nil
}

func printDiffLines(bold *color.Color, lines []string) {
Expand Down
4 changes: 2 additions & 2 deletions go/cmd/dolt/commands/merge.go
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ func executeFFMerge(ctx context.Context, squash bool, dEnv *env.DoltEnv, cm2 *do
}
}

unstagedDocs, err := actions.GetUnstagedDocs(ctx, dEnv)
unstagedDocs, err := actions.GetUnstagedDocs(ctx, dEnv.DbData())
if err != nil {
return errhand.BuildDError("error: unable to determine unstaged docs").AddCause(err).Build()
}
Expand Down Expand Up @@ -401,7 +401,7 @@ func mergedRootToWorking(ctx context.Context, squash bool, dEnv *env.DoltEnv, me
}
}

unstagedDocs, err := actions.GetUnstagedDocs(ctx, dEnv)
unstagedDocs, err := actions.GetUnstagedDocs(ctx, dEnv.DbData())
if err != nil {
return errhand.BuildDError("error: failed to determine unstaged docs").AddCause(err).Build()
}
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/reset.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ func (cmd ResetCmd) Exec(ctx context.Context, commandStr string, args []string,
} else if apr.Contains(HardResetParam) {
err = actions.ResetHard(ctx, dEnv, apr, workingRoot, stagedRoot, headRoot)
} else {
stagedRoot, err = actions.ResetSoft(ctx, dEnv, apr, stagedRoot, headRoot)
stagedRoot, err = actions.ResetSoft(ctx, dEnv.DbData(), apr, stagedRoot, headRoot)

if err != nil {
return handleResetError(err, usage)
Expand Down
174 changes: 174 additions & 0 deletions go/libraries/doltcore/diff/docs_diff_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
// Copyright 2020 Dolthub, Inc.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might as well update this now

GoLand can automate for you

//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package diff

import (
"context"
"testing"

"github.com/stretchr/testify/assert"

"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdocs"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/encoding"
"github.com/dolthub/dolt/go/store/types"
)

func TestDocDiff(t *testing.T) {
ctx := context.Background()
ddb, _ := doltdb.LoadDoltDB(ctx, types.Format_7_18, doltdb.InMemDoltDB)
ddb.WriteEmptyRepo(ctx, "billy bob", "bigbillieb@fake.horse")

cs, _ := doltdb.NewCommitSpec("master")
cm, _ := ddb.Resolve(ctx, cs, nil)

root, err := cm.GetRootValue()
assert.NoError(t, err)

docs := doltdocs.Docs{
{DocPk: doltdb.LicensePk},
{DocPk: doltdb.ReadmePk},
}

// DocsDiff between a root and itself should return no added, modified or removed docs.
added, modified, removed, err := DocsDiff(ctx, root, root, docs)
assert.NoError(t, err)

if len(added)+len(modified)+len(removed) != 0 {
t.Error("Bad doc diff when comparing two repos")
}

// Create tbl1 with one license row
sch := createTestDocsSchema()
licRow := makeDocRow(t, sch, doltdb.LicensePk, types.String("license row"))
m, _ := createTestRows(t, ddb.ValueReadWriter(), sch, []row.Row{licRow})
tbl1, err := CreateTestTable(ddb.ValueReadWriter(), sch, m)
assert.NoError(t, err)

// Create root2 with tbl1 on it (one doc: license)
root2, err := root.PutTable(ctx, doltdb.DocTableName, tbl1)
assert.NoError(t, err)

// DocsDiff between root and root2 should return one added doc, LICENSE.md
added, modified, removed, err = DocsDiff(ctx, root, root2, docs)
assert.NoError(t, err)

if len(added) != 1 || added[0] != "LICENSE.md" || len(modified)+len(removed) != 0 {
t.Error("Bad table diff after adding a single table")
}

// Create tbl2 with one readme row
readmeRow := makeDocRow(t, sch, doltdb.ReadmePk, types.String("readme row"))
m, _ = createTestRows(t, ddb.ValueReadWriter(), sch, []row.Row{readmeRow})
tbl2, err := CreateTestTable(ddb.ValueReadWriter(), sch, m)
assert.NoError(t, err)

// Create root3 with tbl2 on it (one doc: readme)
root3, err := root.PutTable(ctx, doltdb.DocTableName, tbl2)
assert.NoError(t, err)

// DocsDiff between root2 and root3 should return one removed doc (license) and one added doc (readme).
added, modified, removed, err = DocsDiff(ctx, root2, root3, docs)
assert.NoError(t, err)

if len(removed) != 1 || removed[0] != "LICENSE.md" || len(added) != 1 || added[0] != "README.md" || len(modified) != 0 {
t.Error("Bad table diff after adding a single table")
}

// Create tbl3 with 2 doc rows (readme, license)
readmeRowUpdated := makeDocRow(t, sch, doltdb.ReadmePk, types.String("a different readme"))
m, _ = createTestRows(t, ddb.ValueReadWriter(), sch, []row.Row{readmeRowUpdated, licRow})
tbl3, err := CreateTestTable(ddb.ValueReadWriter(), sch, m)
assert.NoError(t, err)

// Create root4 with tbl3 on it (two docs: readme and license)
root4, err := root3.PutTable(ctx, doltdb.DocTableName, tbl3)
assert.NoError(t, err)

// DocsDiff between root3 and root4 should return one added doc (license) and one modified doc (readme).
added, modified, removed, err = DocsDiff(ctx, root3, root4, nil)
assert.NoError(t, err)

if len(added) != 1 || added[0] != "LICENSE.md" || len(modified) != 1 || modified[0] != "README.md" || len(removed) != 0 {
t.Error("Bad table diff after adding a single table")
}

// DocsDiff between root4 and root shows 2 remove docs (license, readme)
added, modified, removed, err = DocsDiff(ctx, root4, root, nil)
assert.NoError(t, err)

if len(removed) != 2 || len(modified) != 0 || len(added) != 0 {
t.Error("Bad table diff after adding a single table")
}
}

func CreateTestTable(vrw types.ValueReadWriter, tSchema schema.Schema, rowData types.Map) (*doltdb.Table, error) {
schemaVal, err := encoding.MarshalSchemaAsNomsValue(context.Background(), vrw, tSchema)

if err != nil {
return nil, err
}

empty, _ := types.NewMap(context.Background(), vrw)
tbl, err := doltdb.NewTable(context.Background(), vrw, schemaVal, rowData, empty)

if err != nil {
return nil, err
}

return tbl, nil
}

func createTestDocsSchema() schema.Schema {
typedColColl, _ := schema.NewColCollection(
schema.NewColumn(doltdb.DocPkColumnName, schema.DocNameTag, types.StringKind, true, schema.NotNullConstraint{}),
schema.NewColumn(doltdb.DocTextColumnName, schema.DocTextTag, types.StringKind, false),
)
sch, err := schema.SchemaFromCols(typedColColl)
if err != nil {
panic(err)
}
return sch
}

func makeDocRow(t *testing.T, sch schema.Schema, pk string, rowVal types.Value) row.Row {
row, err := row.New(types.Format_7_18, sch, row.TaggedValues{
schema.DocNameTag: types.String(pk),
schema.DocTextTag: rowVal,
})
assert.NoError(t, err)

return row
}

func createTestRows(t *testing.T, vrw types.ValueReadWriter, sch schema.Schema, rows []row.Row) (types.Map, []row.Row) {
ctx := context.Background()
var err error

m, err := types.NewMap(ctx, vrw)
assert.NoError(t, err)
ed := m.Edit()

for _, r := range rows {
ed = ed.Set(r.NomsMapKey(sch), r.NomsMapValue(sch))
}

m, err = ed.Map(ctx)
assert.NoError(t, err)

return m, rows
}