Skip to content

Commit

Permalink
Remove the content-mismatch fixer (#4193)
Browse files Browse the repository at this point in the history 
This fixer was used for a bug from 2018 and was very specific. I don't
see any reason to keep it any longer.
  • Loading branch information
@nono
nono committed Oct 25, 2023
2 parents 3004306 + af55ca6 commit 3aaba0e
Show file tree
Hide file tree
Showing 6 changed files with 0 additions and 299 deletions.
44 changes: 0 additions & 44 deletions cmd/fix.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
var (
dryRunFlag bool
withMetadataFlag bool
noDryRunFlag bool
)

var fixerCmdGroup = &cobra.Command{
Expand Down Expand Up @@ -271,47 +270,6 @@ var contactEmailsFixer = &cobra.Command{
},
}

var contentMismatch64Kfixer = &cobra.Command{
Use: "content-mismatch <domain>",
Short: "Fix the content mismatch differences for 64K issue",
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return cmd.Usage()
}

domain := args[0]

buf := new(bytes.Buffer)
body := struct {
DryRun bool `json:"dry_run"`
}{
DryRun: !noDryRunFlag,
}

if err := json.NewEncoder(buf).Encode(body); err != nil {
return err
}

c := newAdminClient()
res, err := c.Req(&request.Options{
Method: "POST",
Path: "/instances/" + url.PathEscape(domain) + "/fixers/content-mismatch",
Body: bytes.NewReader(buf.Bytes()),
})
if err != nil {
return err
}

out, err := io.ReadAll(res.Body)
if err != nil {
return err
}
fmt.Println(string(out))

return nil
},
}

var passwordDefinedFixer = &cobra.Command{
Use: "password-defined <domain>",
Short: "Set the password_defined setting",
Expand Down Expand Up @@ -417,14 +375,12 @@ this instance are correctly set.
func init() {
thumbnailsFixer.Flags().BoolVar(&dryRunFlag, "dry-run", false, "Dry run")
thumbnailsFixer.Flags().BoolVar(&withMetadataFlag, "with-metadata", false, "Recalculate images metadata")
contentMismatch64Kfixer.Flags().BoolVar(&noDryRunFlag, "no-dry-run", false, "Do not dry run")

fixerCmdGroup.AddCommand(jobsFixer)
fixerCmdGroup.AddCommand(mimeFixerCmd)
fixerCmdGroup.AddCommand(redisFixer)
fixerCmdGroup.AddCommand(thumbnailsFixer)
fixerCmdGroup.AddCommand(contactEmailsFixer)
fixerCmdGroup.AddCommand(contentMismatch64Kfixer)
fixerCmdGroup.AddCommand(passwordDefinedFixer)
fixerCmdGroup.AddCommand(orphanAccountFixer)
fixerCmdGroup.AddCommand(serviceTriggersFixer)
Expand Down
1 change: 0 additions & 1 deletion docs/cli/cozy-stack_fix.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ cozy-stack fix <command> [flags]

* [cozy-stack](cozy-stack.md) - cozy-stack is the main command
* [cozy-stack fix contact-emails](cozy-stack_fix_contact-emails.md) - Detect and try to fix invalid emails on contacts
* [cozy-stack fix content-mismatch](cozy-stack_fix_content-mismatch.md) - Fix the content mismatch differences for 64K issue
* [cozy-stack fix indexes](cozy-stack_fix_indexes.md) - Rebuild the CouchDB views and indexes
* [cozy-stack fix jobs](cozy-stack_fix_jobs.md) - Take a look at the consistency of the jobs
* [cozy-stack fix mime](cozy-stack_fix_mime.md) - Fix the class computed from the mime-type
Expand Down
29 changes: 0 additions & 29 deletions docs/cli/cozy-stack_fix_content-mismatch.md
View file

This file was deleted.

35 changes: 0 additions & 35 deletions scripts/completion/cozy-stack.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1933,40 +1933,6 @@ _cozy-stack_fix_contact-emails()
noun_aliases=()
}

_cozy-stack_fix_content-mismatch()
{
last_command="cozy-stack_fix_content-mismatch"

command_aliases=()

commands=()

flags=()
two_word_flags=()
local_nonpersistent_flags=()
flags_with_completion=()
flags_completion=()

flags+=("--no-dry-run")
local_nonpersistent_flags+=("--no-dry-run")
flags+=("--admin-host=")
two_word_flags+=("--admin-host")
flags+=("--admin-port=")
two_word_flags+=("--admin-port")
flags+=("--config=")
two_word_flags+=("--config")
two_word_flags+=("-c")
flags+=("--host=")
two_word_flags+=("--host")
flags+=("--port=")
two_word_flags+=("--port")
two_word_flags+=("-p")

must_have_one_flag=()
must_have_one_noun=()
noun_aliases=()
}

_cozy-stack_fix_indexes()
{
last_command="cozy-stack_fix_indexes"
Expand Down Expand Up @@ -2235,7 +2201,6 @@ _cozy-stack_fix()

commands=()
commands+=("contact-emails")
commands+=("content-mismatch")
commands+=("indexes")
commands+=("jobs")
commands+=("mime")
Expand Down
189 changes: 0 additions & 189 deletions web/instances/fixers.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
package instances

import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"

"github.com/cozy/cozy-stack/model/account"
Expand All @@ -16,196 +12,11 @@ import (
"github.com/cozy/cozy-stack/model/instance/lifecycle"
"github.com/cozy/cozy-stack/model/job"
"github.com/cozy/cozy-stack/model/stack"
"github.com/cozy/cozy-stack/model/vfs"
"github.com/cozy/cozy-stack/pkg/consts"
"github.com/cozy/cozy-stack/pkg/couchdb"
"github.com/labstack/echo/v4"
)

type mismatchStruct struct {
SizeIndex int64 `json:"size_index"`
SizeFile int64 `json:"size_file"`
}

// resEntry contains an out entry of a 64k content mismatch
type resEntry struct {
FilePath string `json:"filepath"`
ID string `json:"id"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}

type resStruct struct {
DryRun bool `json:"dry_run"`
Updated []resEntry `json:"updated"`
Removed []resEntry `json:"removed"`
Domain string `json:"domain"`
}

// contentMismatchFixer fixes the 64k bug
func contentMismatchFixer(c echo.Context) error {
domain := c.Param("domain")
inst, err := lifecycle.GetInstance(domain)
if err != nil {
return fmt.Errorf("Cannot find instance %s", domain)
}

body := struct {
DryRun bool `json:"dry_run"`
}{
DryRun: true,
}

// Try to get the dry_run param from the body. If there is no body, ignore
// it
_ = json.NewDecoder(c.Request().Body).Decode(&body)

// Get the FSCK data from the instance
buf, err := getFSCK(inst)
if err != nil {
return err
}

var content map[string]interface{}
res := &resStruct{
Domain: domain,
DryRun: body.DryRun,
Removed: []resEntry{},
Updated: []resEntry{},
}

scanner := bufio.NewScanner(buf)
for scanner.Scan() {
err = json.NewDecoder(bytes.NewReader(scanner.Bytes())).Decode(&content)
if err != nil {
return err
}

// Filtering the 64kb mismatch issue
if content["type"] != "content_mismatch" {
continue
}

// Prepare the struct & ensure the data should be fixed
contentMismatch, err := prepareMismatchStruct(content)
if err != nil {
return err
}
if !is64ContentMismatch(contentMismatch) {
continue
}

// Finally, fixing the file
err = fixFile(content, contentMismatch, inst, res, body.DryRun)
if err != nil {
return err
}
}

return c.JSON(http.StatusOK, res)
}

func getFSCK(inst *instance.Instance) (io.Reader, error) {
buf := new(bytes.Buffer)
encoder := json.NewEncoder(buf)

logCh := make(chan *vfs.FsckLog)
go func() {
fs := inst.VFS()
_ = fs.Fsck(func(log *vfs.FsckLog) { logCh <- log }, false)
close(logCh)
}()

for log := range logCh {
if !log.IsFile && !log.IsVersion && log.DirDoc != nil {
log.DirDoc.DirsChildren = nil
log.DirDoc.FilesChildren = nil
}
if errenc := encoder.Encode(log); errenc != nil {
return nil, errenc
}
}

return buf, nil
}

func prepareMismatchStruct(content map[string]interface{}) (*mismatchStruct, error) {
contentMismatch := &mismatchStruct{}
marshaled, _ := json.Marshal(content["content_mismatch"])
if err := json.Unmarshal(marshaled, &contentMismatch); err != nil {
return nil, err
}

return contentMismatch, nil
}

// is64ContentMismatch ensures we are treating a 64k content mismatch
func is64ContentMismatch(contentMismatch *mismatchStruct) bool {
// SizeFile should be a multiple of 64k shorter than SizeIndex
size := int64(64 * 1024)

isSmallFile := contentMismatch.SizeIndex <= size && contentMismatch.SizeFile == 0
isMultiple64 := (contentMismatch.SizeIndex-contentMismatch.SizeFile)%size == 0

return isMultiple64 || isSmallFile
}

// fixFile fixes a content-mismatch file
// Trashed:
// - Removes it if the file
// Not Trashed:
// - Appending a corrupted suffix to the file
// - Force the file index size to the real file size
func fixFile(content map[string]interface{}, contentMismatch *mismatchStruct, inst *instance.Instance, res *resStruct, dryRun bool) error {
corruptedSuffix := "-corrupted"

// Removes/update
fileDoc := content["file_doc"].(map[string]interface{})

doc := &vfs.FileDoc{}
err := couchdb.GetDoc(inst, consts.Files, fileDoc["_id"].(string), doc)
if err != nil {
return err
}
instanceVFS := inst.VFS()

// File is trashed
if fileDoc["restore_path"] != nil {
// This is a trashed file, just delete it
res.Removed = append(res.Removed, resEntry{
ID: fileDoc["_id"].(string),
FilePath: fileDoc["path"].(string),
CreatedAt: doc.CreatedAt.String(),
UpdatedAt: doc.UpdatedAt.String(),
})

if !dryRun {
return instanceVFS.DestroyFile(doc)
}
return nil
}

// File is not trashed, updating it
newFileDoc := doc.Clone().(*vfs.FileDoc)

newFileDoc.DocName = doc.DocName + corruptedSuffix
newFileDoc.ByteSize = contentMismatch.SizeFile

res.Updated = append(res.Updated, resEntry{
ID: fileDoc["_id"].(string),
FilePath: fileDoc["path"].(string),
CreatedAt: doc.CreatedAt.String(),
UpdatedAt: doc.UpdatedAt.String(),
})
if !dryRun {
// Let the UpdateFileDoc handles the file doc update. For swift
// layout V1, the file should also be renamed
return instanceVFS.UpdateFileDoc(doc, newFileDoc)
}

return nil
}

func passwordDefinedFixer(c echo.Context) error {
domain := c.Param("domain")
inst, err := lifecycle.GetInstance(domain)
Expand Down
1 change: 0 additions & 1 deletion web/instances/instances.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -678,7 +678,6 @@ func Routes(router *echo.Group) {
router.POST("/:domain/checks/sharings", checkSharings)

// Fixers
router.POST("/:domain/fixers/content-mismatch", contentMismatchFixer)
router.POST("/:domain/fixers/password-defined", passwordDefinedFixer)
router.POST("/:domain/fixers/orphan-account", orphanAccountFixer)
router.POST("/:domain/fixers/service-triggers", serviceTriggersFixer)
Expand Down

0 comments on commit 3aaba0e

Please sign in to comment.